OLD | NEW |
1 // Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 library yaml.scanner; | 5 library yaml.scanner; |
6 | 6 |
7 import 'package:collection/collection.dart'; | 7 import 'package:collection/collection.dart'; |
8 import 'package:string_scanner/string_scanner.dart'; | 8 import 'package:string_scanner/string_scanner.dart'; |
9 import 'package:source_span/source_span.dart'; | 9 import 'package:source_span/source_span.dart'; |
10 | 10 |
(...skipping 274 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
285 /// If so, this sets the scanner's last match to that indicator. | 285 /// If so, this sets the scanner's last match to that indicator. |
286 bool get _isDocumentIndicator { | 286 bool get _isDocumentIndicator { |
287 return _scanner.column == 0 && _isBlankOrEndAt(3) && | 287 return _scanner.column == 0 && _isBlankOrEndAt(3) && |
288 (_scanner.matches('---') || _scanner.matches('...')); | 288 (_scanner.matches('---') || _scanner.matches('...')); |
289 } | 289 } |
290 | 290 |
291 /// Creates a scanner that scans [source]. | 291 /// Creates a scanner that scans [source]. |
292 /// | 292 /// |
293 /// [sourceUrl] can be a String or a [Uri]. | 293 /// [sourceUrl] can be a String or a [Uri]. |
294 Scanner(String source, {sourceUrl}) | 294 Scanner(String source, {sourceUrl}) |
295 : _scanner = new SpanScanner(source, sourceUrl: sourceUrl); | 295 : _scanner = new SpanScanner.eager(source, sourceUrl: sourceUrl); |
296 | 296 |
297 /// Consumes and returns the next token. | 297 /// Consumes and returns the next token. |
298 Token scan() { | 298 Token scan() { |
299 if (_streamEndProduced) throw new StateError("Out of tokens."); | 299 if (_streamEndProduced) throw new StateError("Out of tokens."); |
300 if (!_tokenAvailable) _fetchMoreTokens(); | 300 if (!_tokenAvailable) _fetchMoreTokens(); |
301 | 301 |
302 var token = _tokens.removeFirst(); | 302 var token = _tokens.removeFirst(); |
303 _tokenAvailable = false; | 303 _tokenAvailable = false; |
304 _tokensParsed++; | 304 _tokensParsed++; |
305 _streamEndProduced = token is Token && | 305 _streamEndProduced = token is Token && |
(...skipping 172 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
478 | 478 |
479 // libyaml requires that all simple keys be a single line and no longer | 479 // libyaml requires that all simple keys be a single line and no longer |
480 // than 1024 characters. However, in section 7.4.2 of the spec | 480 // than 1024 characters. However, in section 7.4.2 of the spec |
481 // (http://yaml.org/spec/1.2/spec.html#id2790832), these restrictions are | 481 // (http://yaml.org/spec/1.2/spec.html#id2790832), these restrictions are |
482 // only applied when the curly braces are omitted. It's difficult to | 482 // only applied when the curly braces are omitted. It's difficult to |
483 // retain enough context to know which keys need to have the restriction | 483 // retain enough context to know which keys need to have the restriction |
484 // placed on them, so for now we go the other direction and allow | 484 // placed on them, so for now we go the other direction and allow |
485 // everything but multiline simple keys in a block context. | 485 // everything but multiline simple keys in a block context. |
486 if (!_inBlockContext) continue; | 486 if (!_inBlockContext) continue; |
487 | 487 |
488 if (key.location.line == _scanner.line) continue; | 488 if (key.line == _scanner.line) continue; |
489 | 489 |
490 if (key.required) { | 490 if (key.required) { |
491 throw new YamlException("Expected ':'.", _scanner.emptySpan); | 491 throw new YamlException("Expected ':'.", _scanner.emptySpan); |
492 } | 492 } |
493 | 493 |
494 _simpleKeys[i] = null; | 494 _simpleKeys[i] = null; |
495 } | 495 } |
496 } | 496 } |
497 | 497 |
498 /// Checks if a simple key may start at the current position and saves it if | 498 /// Checks if a simple key may start at the current position and saves it if |
499 /// so. | 499 /// so. |
500 void _saveSimpleKey() { | 500 void _saveSimpleKey() { |
501 // A simple key is required at the current position if the scanner is in the | 501 // A simple key is required at the current position if the scanner is in the |
502 // block context and the current column coincides with the indentation | 502 // block context and the current column coincides with the indentation |
503 // level. | 503 // level. |
504 var required = _inBlockContext && _indent == _scanner.column; | 504 var required = _inBlockContext && _indent == _scanner.column; |
505 | 505 |
506 // A simple key is required only when it is the first token in the current | 506 // A simple key is required only when it is the first token in the current |
507 // line. Therefore it is always allowed. But we add a check anyway. | 507 // line. Therefore it is always allowed. But we add a check anyway. |
508 assert(_simpleKeyAllowed || !required); | 508 assert(_simpleKeyAllowed || !required); |
509 | 509 |
510 if (!_simpleKeyAllowed) return; | 510 if (!_simpleKeyAllowed) return; |
511 | 511 |
512 // If the current position may start a simple key, save it. | 512 // If the current position may start a simple key, save it. |
513 _removeSimpleKey(); | 513 _removeSimpleKey(); |
514 _simpleKeys[_simpleKeys.length - 1] = new _SimpleKey( | 514 _simpleKeys[_simpleKeys.length - 1] = new _SimpleKey( |
515 _tokensParsed + _tokens.length, | 515 _tokensParsed + _tokens.length, |
| 516 _scanner.line, |
| 517 _scanner.column, |
516 _scanner.location, | 518 _scanner.location, |
517 required: required); | 519 required: required); |
518 } | 520 } |
519 | 521 |
520 /// Removes a potential simple key at the current flow level. | 522 /// Removes a potential simple key at the current flow level. |
521 void _removeSimpleKey() { | 523 void _removeSimpleKey() { |
522 var key = _simpleKeys.last; | 524 var key = _simpleKeys.last; |
523 if (key != null && key.required) { | 525 if (key != null && key.required) { |
524 throw new YamlException("Could not find expected ':' for simple key.", | 526 throw new YamlException("Could not find expected ':' for simple key.", |
525 key.location.pointSpan()); | 527 key.location.pointSpan()); |
(...skipping 127 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
653 if (_inBlockContext) { | 655 if (_inBlockContext) { |
654 if (!_simpleKeyAllowed) { | 656 if (!_simpleKeyAllowed) { |
655 throw new YamlException( | 657 throw new YamlException( |
656 "Block sequence entries are not allowed here.", | 658 "Block sequence entries are not allowed here.", |
657 _scanner.emptySpan); | 659 _scanner.emptySpan); |
658 } | 660 } |
659 | 661 |
660 _rollIndent( | 662 _rollIndent( |
661 _scanner.column, | 663 _scanner.column, |
662 TokenType.BLOCK_SEQUENCE_START, | 664 TokenType.BLOCK_SEQUENCE_START, |
663 _scanner.emptySpan.start); | 665 _scanner.location); |
664 } else { | 666 } else { |
665 // It is an error for the '-' indicator to occur in the flow context, but | 667 // It is an error for the '-' indicator to occur in the flow context, but |
666 // we let the Parser detect and report it because it's able to point to | 668 // we let the Parser detect and report it because it's able to point to |
667 // the context. | 669 // the context. |
668 } | 670 } |
669 | 671 |
670 _removeSimpleKey(); | 672 _removeSimpleKey(); |
671 _simpleKeyAllowed = true; | 673 _simpleKeyAllowed = true; |
672 _addCharToken(TokenType.BLOCK_ENTRY); | 674 _addCharToken(TokenType.BLOCK_ENTRY); |
673 } | 675 } |
674 | 676 |
675 /// Produces the [TokenType.KEY] token. | 677 /// Produces the [TokenType.KEY] token. |
676 void _fetchKey() { | 678 void _fetchKey() { |
677 if (_inBlockContext) { | 679 if (_inBlockContext) { |
678 if (!_simpleKeyAllowed) { | 680 if (!_simpleKeyAllowed) { |
679 throw new YamlException("Mapping keys are not allowed here.", | 681 throw new YamlException("Mapping keys are not allowed here.", |
680 _scanner.emptySpan); | 682 _scanner.emptySpan); |
681 } | 683 } |
682 | 684 |
683 _rollIndent( | 685 _rollIndent( |
684 _scanner.column, | 686 _scanner.column, |
685 TokenType.BLOCK_MAPPING_START, | 687 TokenType.BLOCK_MAPPING_START, |
686 _scanner.emptySpan.start); | 688 _scanner.location); |
687 } | 689 } |
688 | 690 |
689 // Simple keys are allowed after `?` in a block context. | 691 // Simple keys are allowed after `?` in a block context. |
690 _simpleKeyAllowed = _inBlockContext; | 692 _simpleKeyAllowed = _inBlockContext; |
691 _addCharToken(TokenType.KEY); | 693 _addCharToken(TokenType.KEY); |
692 } | 694 } |
693 | 695 |
694 /// Produces the [TokenType.VALUE] token. | 696 /// Produces the [TokenType.VALUE] token. |
695 void _fetchValue() { | 697 void _fetchValue() { |
696 var simpleKey = _simpleKeys.last; | 698 var simpleKey = _simpleKeys.last; |
697 if (simpleKey != null) { | 699 if (simpleKey != null) { |
698 // Add a [TokenType.KEY] directive before the first token of the simple | 700 // Add a [TokenType.KEY] directive before the first token of the simple |
699 // key so the parser knows that it's part of a key/value pair. | 701 // key so the parser knows that it's part of a key/value pair. |
700 _tokens.insert(simpleKey.tokenNumber - _tokensParsed, | 702 _tokens.insert(simpleKey.tokenNumber - _tokensParsed, |
701 new Token(TokenType.KEY, simpleKey.location.pointSpan())); | 703 new Token(TokenType.KEY, simpleKey.location.pointSpan())); |
702 | 704 |
703 // In the block context, we may need to add the | 705 // In the block context, we may need to add the |
704 // [TokenType.BLOCK_MAPPING_START] token. | 706 // [TokenType.BLOCK_MAPPING_START] token. |
705 _rollIndent( | 707 _rollIndent( |
706 simpleKey.location.column, | 708 simpleKey.column, |
707 TokenType.BLOCK_MAPPING_START, | 709 TokenType.BLOCK_MAPPING_START, |
708 simpleKey.location, | 710 simpleKey.location, |
709 tokenNumber: simpleKey.tokenNumber); | 711 tokenNumber: simpleKey.tokenNumber); |
710 | 712 |
711 // Remove the simple key. | 713 // Remove the simple key. |
712 _simpleKeys[_simpleKeys.length - 1] = null; | 714 _simpleKeys[_simpleKeys.length - 1] = null; |
713 | 715 |
714 // A simple key cannot follow another simple key. | 716 // A simple key cannot follow another simple key. |
715 _simpleKeyAllowed = false; | 717 _simpleKeyAllowed = false; |
716 } else if (_inBlockContext) { | 718 } else if (_inBlockContext) { |
(...skipping 915 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1632 /// This is the index relative to all tokens emitted, rather than relative to | 1634 /// This is the index relative to all tokens emitted, rather than relative to |
1633 /// [_tokens]. | 1635 /// [_tokens]. |
1634 final int tokenNumber; | 1636 final int tokenNumber; |
1635 | 1637 |
1636 /// The source location of the beginning of the simple key. | 1638 /// The source location of the beginning of the simple key. |
1637 /// | 1639 /// |
1638 /// This is used for error reporting and for determining when a simple key is | 1640 /// This is used for error reporting and for determining when a simple key is |
1639 /// no longer on the current line. | 1641 /// no longer on the current line. |
1640 final SourceLocation location; | 1642 final SourceLocation location; |
1641 | 1643 |
| 1644 /// The line on which the key appears. |
| 1645 /// |
| 1646 /// We could get this from [location], but that requires a binary search |
| 1647 /// whereas this is O(1). |
| 1648 final int line; |
| 1649 |
| 1650 /// The column on which the key appears. |
| 1651 /// |
| 1652 /// We could get this from [location], but that requires a binary search |
| 1653 /// whereas this is O(1). |
| 1654 final int column; |
| 1655 |
1642 /// Whether this key must exist for the document to be scanned. | 1656 /// Whether this key must exist for the document to be scanned. |
1643 final bool required; | 1657 final bool required; |
1644 | 1658 |
1645 _SimpleKey(this.tokenNumber, this.location, {bool required}) | 1659 _SimpleKey(this.tokenNumber, this.line, this.column, this.location, |
| 1660 {bool required}) |
1646 : required = required; | 1661 : required = required; |
1647 } | 1662 } |
1648 | 1663 |
1649 /// An enum of chomping indicators that describe how to handle trailing | 1664 /// An enum of chomping indicators that describe how to handle trailing |
1650 /// whitespace for a block scalar. | 1665 /// whitespace for a block scalar. |
1651 /// | 1666 /// |
1652 /// See http://yaml.org/spec/1.2/spec.html#id2794534. | 1667 /// See http://yaml.org/spec/1.2/spec.html#id2794534. |
1653 class _Chomping { | 1668 class _Chomping { |
1654 /// All trailing whitespace is discarded. | 1669 /// All trailing whitespace is discarded. |
1655 static const STRIP = const _Chomping("STRIP"); | 1670 static const STRIP = const _Chomping("STRIP"); |
1656 | 1671 |
1657 /// A single trailing newline is retained. | 1672 /// A single trailing newline is retained. |
1658 static const CLIP = const _Chomping("CLIP"); | 1673 static const CLIP = const _Chomping("CLIP"); |
1659 | 1674 |
1660 /// All trailing whitespace is preserved. | 1675 /// All trailing whitespace is preserved. |
1661 static const KEEP = const _Chomping("KEEP"); | 1676 static const KEEP = const _Chomping("KEEP"); |
1662 | 1677 |
1663 final String name; | 1678 final String name; |
1664 | 1679 |
1665 const _Chomping(this.name); | 1680 const _Chomping(this.name); |
1666 | 1681 |
1667 String toString() => name; | 1682 String toString() => name; |
1668 } | 1683 } |
OLD | NEW |