Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(403)

Side by Side Diff: lib/src/scanner.dart

Issue 1329763002: Don't use regular expressions in Loader. (Closed) Base URL: git@github.com:dart-lang/yaml@master
Patch Set: Created 5 years, 3 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 // Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file 1 // Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file
2 // for details. All rights reserved. Use of this source code is governed by a 2 // for details. All rights reserved. Use of this source code is governed by a
3 // BSD-style license that can be found in the LICENSE file. 3 // BSD-style license that can be found in the LICENSE file.
4 4
5 library yaml.scanner; 5 library yaml.scanner;
6 6
7 import 'package:collection/collection.dart'; 7 import 'package:collection/collection.dart';
8 import 'package:string_scanner/string_scanner.dart'; 8 import 'package:string_scanner/string_scanner.dart';
9 import 'package:source_span/source_span.dart'; 9 import 'package:source_span/source_span.dart';
10 10
(...skipping 274 matching lines...) Expand 10 before | Expand all | Expand 10 after
285 /// If so, this sets the scanner's last match to that indicator. 285 /// If so, this sets the scanner's last match to that indicator.
286 bool get _isDocumentIndicator { 286 bool get _isDocumentIndicator {
287 return _scanner.column == 0 && _isBlankOrEndAt(3) && 287 return _scanner.column == 0 && _isBlankOrEndAt(3) &&
288 (_scanner.matches('---') || _scanner.matches('...')); 288 (_scanner.matches('---') || _scanner.matches('...'));
289 } 289 }
290 290
291 /// Creates a scanner that scans [source]. 291 /// Creates a scanner that scans [source].
292 /// 292 ///
293 /// [sourceUrl] can be a String or a [Uri]. 293 /// [sourceUrl] can be a String or a [Uri].
294 Scanner(String source, {sourceUrl}) 294 Scanner(String source, {sourceUrl})
295 : _scanner = new SpanScanner.eager(source, sourceUrl: sourceUrl); 295 : _scanner = new SpanScanner(source, sourceUrl: sourceUrl);
Bob Nystrom 2015/09/02 23:13:14 Do you need to merge in that previous change?
nweiz 2015/09/02 23:37:59 Oops!
296 296
297 /// Consumes and returns the next token. 297 /// Consumes and returns the next token.
298 Token scan() { 298 Token scan() {
299 if (_streamEndProduced) throw new StateError("Out of tokens."); 299 if (_streamEndProduced) throw new StateError("Out of tokens.");
300 if (!_tokenAvailable) _fetchMoreTokens(); 300 if (!_tokenAvailable) _fetchMoreTokens();
301 301
302 var token = _tokens.removeFirst(); 302 var token = _tokens.removeFirst();
303 _tokenAvailable = false; 303 _tokenAvailable = false;
304 _tokensParsed++; 304 _tokensParsed++;
305 _streamEndProduced = token is Token && 305 _streamEndProduced = token is Token &&
(...skipping 172 matching lines...) Expand 10 before | Expand all | Expand 10 after
478 478
479 // libyaml requires that all simple keys be a single line and no longer 479 // libyaml requires that all simple keys be a single line and no longer
480 // than 1024 characters. However, in section 7.4.2 of the spec 480 // than 1024 characters. However, in section 7.4.2 of the spec
481 // (http://yaml.org/spec/1.2/spec.html#id2790832), these restrictions are 481 // (http://yaml.org/spec/1.2/spec.html#id2790832), these restrictions are
482 // only applied when the curly braces are omitted. It's difficult to 482 // only applied when the curly braces are omitted. It's difficult to
483 // retain enough context to know which keys need to have the restriction 483 // retain enough context to know which keys need to have the restriction
484 // placed on them, so for now we go the other direction and allow 484 // placed on them, so for now we go the other direction and allow
485 // everything but multiline simple keys in a block context. 485 // everything but multiline simple keys in a block context.
486 if (!_inBlockContext) continue; 486 if (!_inBlockContext) continue;
487 487
488 if (key.line == _scanner.line) continue; 488 if (key.location.line == _scanner.line) continue;
489 489
490 if (key.required) { 490 if (key.required) {
491 throw new YamlException("Expected ':'.", _scanner.emptySpan); 491 throw new YamlException("Expected ':'.", _scanner.emptySpan);
492 } 492 }
493 493
494 _simpleKeys[i] = null; 494 _simpleKeys[i] = null;
495 } 495 }
496 } 496 }
497 497
498 /// Checks if a simple key may start at the current position and saves it if 498 /// Checks if a simple key may start at the current position and saves it if
499 /// so. 499 /// so.
500 void _saveSimpleKey() { 500 void _saveSimpleKey() {
501 // A simple key is required at the current position if the scanner is in the 501 // A simple key is required at the current position if the scanner is in the
502 // block context and the current column coincides with the indentation 502 // block context and the current column coincides with the indentation
503 // level. 503 // level.
504 var required = _inBlockContext && _indent == _scanner.column; 504 var required = _inBlockContext && _indent == _scanner.column;
505 505
506 // A simple key is required only when it is the first token in the current 506 // A simple key is required only when it is the first token in the current
507 // line. Therefore it is always allowed. But we add a check anyway. 507 // line. Therefore it is always allowed. But we add a check anyway.
508 assert(_simpleKeyAllowed || !required); 508 assert(_simpleKeyAllowed || !required);
509 509
510 if (!_simpleKeyAllowed) return; 510 if (!_simpleKeyAllowed) return;
511 511
512 // If the current position may start a simple key, save it. 512 // If the current position may start a simple key, save it.
513 _removeSimpleKey(); 513 _removeSimpleKey();
514 _simpleKeys[_simpleKeys.length - 1] = new _SimpleKey( 514 _simpleKeys[_simpleKeys.length - 1] = new _SimpleKey(
515 _tokensParsed + _tokens.length, 515 _tokensParsed + _tokens.length,
516 _scanner.line,
517 _scanner.column,
518 _scanner.location, 516 _scanner.location,
519 required: required); 517 required: required);
520 } 518 }
521 519
522 /// Removes a potential simple key at the current flow level. 520 /// Removes a potential simple key at the current flow level.
523 void _removeSimpleKey() { 521 void _removeSimpleKey() {
524 var key = _simpleKeys.last; 522 var key = _simpleKeys.last;
525 if (key != null && key.required) { 523 if (key != null && key.required) {
526 throw new YamlException("Could not find expected ':' for simple key.", 524 throw new YamlException("Could not find expected ':' for simple key.",
527 key.location.pointSpan()); 525 key.location.pointSpan());
(...skipping 127 matching lines...) Expand 10 before | Expand all | Expand 10 after
655 if (_inBlockContext) { 653 if (_inBlockContext) {
656 if (!_simpleKeyAllowed) { 654 if (!_simpleKeyAllowed) {
657 throw new YamlException( 655 throw new YamlException(
658 "Block sequence entries are not allowed here.", 656 "Block sequence entries are not allowed here.",
659 _scanner.emptySpan); 657 _scanner.emptySpan);
660 } 658 }
661 659
662 _rollIndent( 660 _rollIndent(
663 _scanner.column, 661 _scanner.column,
664 TokenType.BLOCK_SEQUENCE_START, 662 TokenType.BLOCK_SEQUENCE_START,
665 _scanner.location); 663 _scanner.emptySpan.start);
666 } else { 664 } else {
667 // It is an error for the '-' indicator to occur in the flow context, but 665 // It is an error for the '-' indicator to occur in the flow context, but
668 // we let the Parser detect and report it because it's able to point to 666 // we let the Parser detect and report it because it's able to point to
669 // the context. 667 // the context.
670 } 668 }
671 669
672 _removeSimpleKey(); 670 _removeSimpleKey();
673 _simpleKeyAllowed = true; 671 _simpleKeyAllowed = true;
674 _addCharToken(TokenType.BLOCK_ENTRY); 672 _addCharToken(TokenType.BLOCK_ENTRY);
675 } 673 }
676 674
677 /// Produces the [TokenType.KEY] token. 675 /// Produces the [TokenType.KEY] token.
678 void _fetchKey() { 676 void _fetchKey() {
679 if (_inBlockContext) { 677 if (_inBlockContext) {
680 if (!_simpleKeyAllowed) { 678 if (!_simpleKeyAllowed) {
681 throw new YamlException("Mapping keys are not allowed here.", 679 throw new YamlException("Mapping keys are not allowed here.",
682 _scanner.emptySpan); 680 _scanner.emptySpan);
683 } 681 }
684 682
685 _rollIndent( 683 _rollIndent(
686 _scanner.column, 684 _scanner.column,
687 TokenType.BLOCK_MAPPING_START, 685 TokenType.BLOCK_MAPPING_START,
688 _scanner.location); 686 _scanner.emptySpan.start);
689 } 687 }
690 688
691 // Simple keys are allowed after `?` in a block context. 689 // Simple keys are allowed after `?` in a block context.
692 _simpleKeyAllowed = _inBlockContext; 690 _simpleKeyAllowed = _inBlockContext;
693 _addCharToken(TokenType.KEY); 691 _addCharToken(TokenType.KEY);
694 } 692 }
695 693
696 /// Produces the [TokenType.VALUE] token. 694 /// Produces the [TokenType.VALUE] token.
697 void _fetchValue() { 695 void _fetchValue() {
698 var simpleKey = _simpleKeys.last; 696 var simpleKey = _simpleKeys.last;
699 if (simpleKey != null) { 697 if (simpleKey != null) {
700 // Add a [TokenType.KEY] directive before the first token of the simple 698 // Add a [TokenType.KEY] directive before the first token of the simple
701 // key so the parser knows that it's part of a key/value pair. 699 // key so the parser knows that it's part of a key/value pair.
702 _tokens.insert(simpleKey.tokenNumber - _tokensParsed, 700 _tokens.insert(simpleKey.tokenNumber - _tokensParsed,
703 new Token(TokenType.KEY, simpleKey.location.pointSpan())); 701 new Token(TokenType.KEY, simpleKey.location.pointSpan()));
704 702
705 // In the block context, we may need to add the 703 // In the block context, we may need to add the
706 // [TokenType.BLOCK_MAPPING_START] token. 704 // [TokenType.BLOCK_MAPPING_START] token.
707 _rollIndent( 705 _rollIndent(
708 simpleKey.column, 706 simpleKey.location.column,
709 TokenType.BLOCK_MAPPING_START, 707 TokenType.BLOCK_MAPPING_START,
710 simpleKey.location, 708 simpleKey.location,
711 tokenNumber: simpleKey.tokenNumber); 709 tokenNumber: simpleKey.tokenNumber);
712 710
713 // Remove the simple key. 711 // Remove the simple key.
714 _simpleKeys[_simpleKeys.length - 1] = null; 712 _simpleKeys[_simpleKeys.length - 1] = null;
715 713
716 // A simple key cannot follow another simple key. 714 // A simple key cannot follow another simple key.
717 _simpleKeyAllowed = false; 715 _simpleKeyAllowed = false;
718 } else if (_inBlockContext) { 716 } else if (_inBlockContext) {
(...skipping 915 matching lines...) Expand 10 before | Expand all | Expand 10 after
1634 /// This is the index relative to all tokens emitted, rather than relative to 1632 /// This is the index relative to all tokens emitted, rather than relative to
1635 /// [_tokens]. 1633 /// [_tokens].
1636 final int tokenNumber; 1634 final int tokenNumber;
1637 1635
1638 /// The source location of the beginning of the simple key. 1636 /// The source location of the beginning of the simple key.
1639 /// 1637 ///
1640 /// This is used for error reporting and for determining when a simple key is 1638 /// This is used for error reporting and for determining when a simple key is
1641 /// no longer on the current line. 1639 /// no longer on the current line.
1642 final SourceLocation location; 1640 final SourceLocation location;
1643 1641
1644 /// The line on which the key appears.
1645 ///
1646 /// We could get this from [location], but that requires a binary search
1647 /// whereas this is O(1).
1648 final int line;
1649
1650 /// The column on which the key appears.
1651 ///
1652 /// We could get this from [location], but that requires a binary search
1653 /// whereas this is O(1).
1654 final int column;
1655
1656 /// Whether this key must exist for the document to be scanned. 1642 /// Whether this key must exist for the document to be scanned.
1657 final bool required; 1643 final bool required;
1658 1644
1659 _SimpleKey(this.tokenNumber, this.line, this.column, this.location, 1645 _SimpleKey(this.tokenNumber, this.location, {bool required})
1660 {bool required})
1661 : required = required; 1646 : required = required;
1662 } 1647 }
1663 1648
1664 /// An enum of chomping indicators that describe how to handle trailing 1649 /// An enum of chomping indicators that describe how to handle trailing
1665 /// whitespace for a block scalar. 1650 /// whitespace for a block scalar.
1666 /// 1651 ///
1667 /// See http://yaml.org/spec/1.2/spec.html#id2794534. 1652 /// See http://yaml.org/spec/1.2/spec.html#id2794534.
1668 class _Chomping { 1653 class _Chomping {
1669 /// All trailing whitespace is discarded. 1654 /// All trailing whitespace is discarded.
1670 static const STRIP = const _Chomping("STRIP"); 1655 static const STRIP = const _Chomping("STRIP");
1671 1656
1672 /// A single trailing newline is retained. 1657 /// A single trailing newline is retained.
1673 static const CLIP = const _Chomping("CLIP"); 1658 static const CLIP = const _Chomping("CLIP");
1674 1659
1675 /// All trailing whitespace is preserved. 1660 /// All trailing whitespace is preserved.
1676 static const KEEP = const _Chomping("KEEP"); 1661 static const KEEP = const _Chomping("KEEP");
1677 1662
1678 final String name; 1663 final String name;
1679 1664
1680 const _Chomping(this.name); 1665 const _Chomping(this.name);
1681 1666
1682 String toString() => name; 1667 String toString() => name;
1683 } 1668 }
OLDNEW
« no previous file with comments | « lib/src/loader.dart ('k') | lib/src/yaml_node.dart » ('j') | pubspec.yaml » ('J')

Powered by Google App Engine
This is Rietveld 408576698