| OLD | NEW |
| 1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 // Generated by scripts/tokenizer_gen.py. | 4 // Generated by scripts/tokenizer_gen.py. |
| 5 | 5 |
| 6 | 6 |
| 7 interface TokenSource { | 7 interface TokenSource { |
| 8 Token next(); | 8 Token next(); |
| 9 } | 9 } |
| 10 | 10 |
| (...skipping 248 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 259 Token _makeStringToken(List<int> buf, bool isPart) { | 259 Token _makeStringToken(List<int> buf, bool isPart) { |
| 260 final s = new String.fromCharCodes(buf); | 260 final s = new String.fromCharCodes(buf); |
| 261 final kind = isPart ? TokenKind.STRING_PART : TokenKind.STRING; | 261 final kind = isPart ? TokenKind.STRING_PART : TokenKind.STRING; |
| 262 return new LiteralToken(kind, _source, _startIndex, _index, s); | 262 return new LiteralToken(kind, _source, _startIndex, _index, s); |
| 263 } | 263 } |
| 264 | 264 |
| 265 Token _makeRawStringToken(bool isMultiline) { | 265 Token _makeRawStringToken(bool isMultiline) { |
| 266 String s; | 266 String s; |
| 267 if (isMultiline) { | 267 if (isMultiline) { |
| 268 // Skip initial newline in multiline strings | 268 // Skip initial newline in multiline strings |
| 269 if (_source.text[_startIndex + 4] == '\n') { | 269 int start = _startIndex + 4; |
| 270 s = _source.text.substring(_startIndex + 5, _index - 3); | 270 if (_source.text[start] == '\n') start++; |
| 271 } else { | 271 s = _source.text.substring(start, _index - 3); |
| 272 s = _source.text.substring(_startIndex + 4, _index - 3); | |
| 273 } | |
| 274 } else { | 272 } else { |
| 275 s = _source.text.substring(_startIndex + 2, _index - 1); | 273 s = _source.text.substring(_startIndex + 2, _index - 1); |
| 276 } | 274 } |
| 277 return new LiteralToken(TokenKind.STRING, _source, _startIndex, _index, s); | 275 return new LiteralToken(TokenKind.STRING, _source, _startIndex, _index, s); |
| 278 } | 276 } |
| 279 | 277 |
| 280 Token finishMultilineString(int quote) { | 278 Token finishMultilineString(int quote) { |
| 281 var buf = new List<int>(); | 279 var buf = <int>[]; |
| 282 while (true) { | 280 while (true) { |
| 283 int ch = _nextChar(); | 281 int ch = _nextChar(); |
| 284 if (ch == 0) { | 282 if (ch == 0) { |
| 285 return _errorToken(); | 283 return _errorToken(); |
| 286 } else if (ch == quote) { | 284 } else if (ch == quote) { |
| 287 if (_maybeEatChar(quote)) { | 285 if (_maybeEatChar(quote)) { |
| 288 if (_maybeEatChar(quote)) { | 286 if (_maybeEatChar(quote)) { |
| 289 return _makeStringToken(buf, false); | 287 return _makeStringToken(buf, false); |
| 290 } | 288 } |
| 291 buf.add(quote); | 289 buf.add(quote); |
| (...skipping 47 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 339 } | 337 } |
| 340 } | 338 } |
| 341 return finishStringBody(quote); | 339 return finishStringBody(quote); |
| 342 } | 340 } |
| 343 | 341 |
| 344 Token finishRawString(int quote) { | 342 Token finishRawString(int quote) { |
| 345 if (_maybeEatChar(quote)) { | 343 if (_maybeEatChar(quote)) { |
| 346 if (_maybeEatChar(quote)) { | 344 if (_maybeEatChar(quote)) { |
| 347 return finishMultilineRawString(quote); | 345 return finishMultilineRawString(quote); |
| 348 } else { | 346 } else { |
| 349 return _makeStringToken(new List<int>(), false); | 347 return _makeStringToken(<int>[], false); |
| 350 } | 348 } |
| 351 } | 349 } |
| 352 while (true) { | 350 while (true) { |
| 353 int ch = _nextChar(); | 351 int ch = _nextChar(); |
| 354 if (ch == quote) { | 352 if (ch == quote) { |
| 355 return _makeRawStringToken(false); | 353 return _makeRawStringToken(false); |
| 356 } else if (ch == 0) { | 354 } else if (ch == 0) { |
| 357 return _errorToken(); | 355 return _errorToken(); |
| 358 } | 356 } |
| 359 } | 357 } |
| (...skipping 115 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 475 } | 473 } |
| 476 } | 474 } |
| 477 int kind = getIdentifierKind(); | 475 int kind = getIdentifierKind(); |
| 478 if (kind == TokenKind.IDENTIFIER) { | 476 if (kind == TokenKind.IDENTIFIER) { |
| 479 return _finishToken(TokenKind.IDENTIFIER); | 477 return _finishToken(TokenKind.IDENTIFIER); |
| 480 } else { | 478 } else { |
| 481 return _finishToken(kind); | 479 return _finishToken(kind); |
| 482 } | 480 } |
| 483 } | 481 } |
| 484 } | 482 } |
| OLD | NEW |