| OLD | NEW |
| 1 // Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 library yaml.scanner; | 5 library yaml.scanner; |
| 6 | 6 |
| 7 import 'package:collection/collection.dart'; | 7 import 'package:collection/collection.dart'; |
| 8 import 'package:string_scanner/string_scanner.dart'; | 8 import 'package:string_scanner/string_scanner.dart'; |
| 9 import 'package:source_span/source_span.dart'; | 9 import 'package:source_span/source_span.dart'; |
| 10 | 10 |
| (...skipping 274 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 285 /// If so, this sets the scanner's last match to that indicator. | 285 /// If so, this sets the scanner's last match to that indicator. |
| 286 bool get _isDocumentIndicator { | 286 bool get _isDocumentIndicator { |
| 287 return _scanner.column == 0 && _isBlankOrEndAt(3) && | 287 return _scanner.column == 0 && _isBlankOrEndAt(3) && |
| 288 (_scanner.matches('---') || _scanner.matches('...')); | 288 (_scanner.matches('---') || _scanner.matches('...')); |
| 289 } | 289 } |
| 290 | 290 |
| 291 /// Creates a scanner that scans [source]. | 291 /// Creates a scanner that scans [source]. |
| 292 /// | 292 /// |
| 293 /// [sourceUrl] can be a String or a [Uri]. | 293 /// [sourceUrl] can be a String or a [Uri]. |
| 294 Scanner(String source, {sourceUrl}) | 294 Scanner(String source, {sourceUrl}) |
| 295 : _scanner = new SpanScanner(source, sourceUrl: sourceUrl); | 295 : _scanner = new SpanScanner.eager(source, sourceUrl: sourceUrl); |
| 296 | 296 |
| 297 /// Consumes and returns the next token. | 297 /// Consumes and returns the next token. |
| 298 Token scan() { | 298 Token scan() { |
| 299 if (_streamEndProduced) throw new StateError("Out of tokens."); | 299 if (_streamEndProduced) throw new StateError("Out of tokens."); |
| 300 if (!_tokenAvailable) _fetchMoreTokens(); | 300 if (!_tokenAvailable) _fetchMoreTokens(); |
| 301 | 301 |
| 302 var token = _tokens.removeFirst(); | 302 var token = _tokens.removeFirst(); |
| 303 _tokenAvailable = false; | 303 _tokenAvailable = false; |
| 304 _tokensParsed++; | 304 _tokensParsed++; |
| 305 _streamEndProduced = token is Token && | 305 _streamEndProduced = token is Token && |
| (...skipping 172 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 478 | 478 |
| 479 // libyaml requires that all simple keys be a single line and no longer | 479 // libyaml requires that all simple keys be a single line and no longer |
| 480 // than 1024 characters. However, in section 7.4.2 of the spec | 480 // than 1024 characters. However, in section 7.4.2 of the spec |
| 481 // (http://yaml.org/spec/1.2/spec.html#id2790832), these restrictions are | 481 // (http://yaml.org/spec/1.2/spec.html#id2790832), these restrictions are |
| 482 // only applied when the curly braces are omitted. It's difficult to | 482 // only applied when the curly braces are omitted. It's difficult to |
| 483 // retain enough context to know which keys need to have the restriction | 483 // retain enough context to know which keys need to have the restriction |
| 484 // placed on them, so for now we go the other direction and allow | 484 // placed on them, so for now we go the other direction and allow |
| 485 // everything but multiline simple keys in a block context. | 485 // everything but multiline simple keys in a block context. |
| 486 if (!_inBlockContext) continue; | 486 if (!_inBlockContext) continue; |
| 487 | 487 |
| 488 if (key.location.line == _scanner.line) continue; | 488 if (key.line == _scanner.line) continue; |
| 489 | 489 |
| 490 if (key.required) { | 490 if (key.required) { |
| 491 throw new YamlException("Expected ':'.", _scanner.emptySpan); | 491 throw new YamlException("Expected ':'.", _scanner.emptySpan); |
| 492 } | 492 } |
| 493 | 493 |
| 494 _simpleKeys[i] = null; | 494 _simpleKeys[i] = null; |
| 495 } | 495 } |
| 496 } | 496 } |
| 497 | 497 |
| 498 /// Checks if a simple key may start at the current position and saves it if | 498 /// Checks if a simple key may start at the current position and saves it if |
| 499 /// so. | 499 /// so. |
| 500 void _saveSimpleKey() { | 500 void _saveSimpleKey() { |
| 501 // A simple key is required at the current position if the scanner is in the | 501 // A simple key is required at the current position if the scanner is in the |
| 502 // block context and the current column coincides with the indentation | 502 // block context and the current column coincides with the indentation |
| 503 // level. | 503 // level. |
| 504 var required = _inBlockContext && _indent == _scanner.column; | 504 var required = _inBlockContext && _indent == _scanner.column; |
| 505 | 505 |
| 506 // A simple key is required only when it is the first token in the current | 506 // A simple key is required only when it is the first token in the current |
| 507 // line. Therefore it is always allowed. But we add a check anyway. | 507 // line. Therefore it is always allowed. But we add a check anyway. |
| 508 assert(_simpleKeyAllowed || !required); | 508 assert(_simpleKeyAllowed || !required); |
| 509 | 509 |
| 510 if (!_simpleKeyAllowed) return; | 510 if (!_simpleKeyAllowed) return; |
| 511 | 511 |
| 512 // If the current position may start a simple key, save it. | 512 // If the current position may start a simple key, save it. |
| 513 _removeSimpleKey(); | 513 _removeSimpleKey(); |
| 514 _simpleKeys[_simpleKeys.length - 1] = new _SimpleKey( | 514 _simpleKeys[_simpleKeys.length - 1] = new _SimpleKey( |
| 515 _tokensParsed + _tokens.length, | 515 _tokensParsed + _tokens.length, |
| 516 _scanner.line, |
| 517 _scanner.column, |
| 516 _scanner.location, | 518 _scanner.location, |
| 517 required: required); | 519 required: required); |
| 518 } | 520 } |
| 519 | 521 |
| 520 /// Removes a potential simple key at the current flow level. | 522 /// Removes a potential simple key at the current flow level. |
| 521 void _removeSimpleKey() { | 523 void _removeSimpleKey() { |
| 522 var key = _simpleKeys.last; | 524 var key = _simpleKeys.last; |
| 523 if (key != null && key.required) { | 525 if (key != null && key.required) { |
| 524 throw new YamlException("Could not find expected ':' for simple key.", | 526 throw new YamlException("Could not find expected ':' for simple key.", |
| 525 key.location.pointSpan()); | 527 key.location.pointSpan()); |
| (...skipping 127 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 653 if (_inBlockContext) { | 655 if (_inBlockContext) { |
| 654 if (!_simpleKeyAllowed) { | 656 if (!_simpleKeyAllowed) { |
| 655 throw new YamlException( | 657 throw new YamlException( |
| 656 "Block sequence entries are not allowed here.", | 658 "Block sequence entries are not allowed here.", |
| 657 _scanner.emptySpan); | 659 _scanner.emptySpan); |
| 658 } | 660 } |
| 659 | 661 |
| 660 _rollIndent( | 662 _rollIndent( |
| 661 _scanner.column, | 663 _scanner.column, |
| 662 TokenType.BLOCK_SEQUENCE_START, | 664 TokenType.BLOCK_SEQUENCE_START, |
| 663 _scanner.emptySpan.start); | 665 _scanner.location); |
| 664 } else { | 666 } else { |
| 665 // It is an error for the '-' indicator to occur in the flow context, but | 667 // It is an error for the '-' indicator to occur in the flow context, but |
| 666 // we let the Parser detect and report it because it's able to point to | 668 // we let the Parser detect and report it because it's able to point to |
| 667 // the context. | 669 // the context. |
| 668 } | 670 } |
| 669 | 671 |
| 670 _removeSimpleKey(); | 672 _removeSimpleKey(); |
| 671 _simpleKeyAllowed = true; | 673 _simpleKeyAllowed = true; |
| 672 _addCharToken(TokenType.BLOCK_ENTRY); | 674 _addCharToken(TokenType.BLOCK_ENTRY); |
| 673 } | 675 } |
| 674 | 676 |
| 675 /// Produces the [TokenType.KEY] token. | 677 /// Produces the [TokenType.KEY] token. |
| 676 void _fetchKey() { | 678 void _fetchKey() { |
| 677 if (_inBlockContext) { | 679 if (_inBlockContext) { |
| 678 if (!_simpleKeyAllowed) { | 680 if (!_simpleKeyAllowed) { |
| 679 throw new YamlException("Mapping keys are not allowed here.", | 681 throw new YamlException("Mapping keys are not allowed here.", |
| 680 _scanner.emptySpan); | 682 _scanner.emptySpan); |
| 681 } | 683 } |
| 682 | 684 |
| 683 _rollIndent( | 685 _rollIndent( |
| 684 _scanner.column, | 686 _scanner.column, |
| 685 TokenType.BLOCK_MAPPING_START, | 687 TokenType.BLOCK_MAPPING_START, |
| 686 _scanner.emptySpan.start); | 688 _scanner.location); |
| 687 } | 689 } |
| 688 | 690 |
| 689 // Simple keys are allowed after `?` in a block context. | 691 // Simple keys are allowed after `?` in a block context. |
| 690 _simpleKeyAllowed = _inBlockContext; | 692 _simpleKeyAllowed = _inBlockContext; |
| 691 _addCharToken(TokenType.KEY); | 693 _addCharToken(TokenType.KEY); |
| 692 } | 694 } |
| 693 | 695 |
| 694 /// Produces the [TokenType.VALUE] token. | 696 /// Produces the [TokenType.VALUE] token. |
| 695 void _fetchValue() { | 697 void _fetchValue() { |
| 696 var simpleKey = _simpleKeys.last; | 698 var simpleKey = _simpleKeys.last; |
| 697 if (simpleKey != null) { | 699 if (simpleKey != null) { |
| 698 // Add a [TokenType.KEY] directive before the first token of the simple | 700 // Add a [TokenType.KEY] directive before the first token of the simple |
| 699 // key so the parser knows that it's part of a key/value pair. | 701 // key so the parser knows that it's part of a key/value pair. |
| 700 _tokens.insert(simpleKey.tokenNumber - _tokensParsed, | 702 _tokens.insert(simpleKey.tokenNumber - _tokensParsed, |
| 701 new Token(TokenType.KEY, simpleKey.location.pointSpan())); | 703 new Token(TokenType.KEY, simpleKey.location.pointSpan())); |
| 702 | 704 |
| 703 // In the block context, we may need to add the | 705 // In the block context, we may need to add the |
| 704 // [TokenType.BLOCK_MAPPING_START] token. | 706 // [TokenType.BLOCK_MAPPING_START] token. |
| 705 _rollIndent( | 707 _rollIndent( |
| 706 simpleKey.location.column, | 708 simpleKey.column, |
| 707 TokenType.BLOCK_MAPPING_START, | 709 TokenType.BLOCK_MAPPING_START, |
| 708 simpleKey.location, | 710 simpleKey.location, |
| 709 tokenNumber: simpleKey.tokenNumber); | 711 tokenNumber: simpleKey.tokenNumber); |
| 710 | 712 |
| 711 // Remove the simple key. | 713 // Remove the simple key. |
| 712 _simpleKeys[_simpleKeys.length - 1] = null; | 714 _simpleKeys[_simpleKeys.length - 1] = null; |
| 713 | 715 |
| 714 // A simple key cannot follow another simple key. | 716 // A simple key cannot follow another simple key. |
| 715 _simpleKeyAllowed = false; | 717 _simpleKeyAllowed = false; |
| 716 } else if (_inBlockContext) { | 718 } else if (_inBlockContext) { |
| (...skipping 915 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1632 /// This is the index relative to all tokens emitted, rather than relative to | 1634 /// This is the index relative to all tokens emitted, rather than relative to |
| 1633 /// [_tokens]. | 1635 /// [_tokens]. |
| 1634 final int tokenNumber; | 1636 final int tokenNumber; |
| 1635 | 1637 |
| 1636 /// The source location of the beginning of the simple key. | 1638 /// The source location of the beginning of the simple key. |
| 1637 /// | 1639 /// |
| 1638 /// This is used for error reporting and for determining when a simple key is | 1640 /// This is used for error reporting and for determining when a simple key is |
| 1639 /// no longer on the current line. | 1641 /// no longer on the current line. |
| 1640 final SourceLocation location; | 1642 final SourceLocation location; |
| 1641 | 1643 |
| 1644 /// The line on which the key appears. |
| 1645 /// |
| 1646 /// We could get this from [location], but that requires a binary search |
| 1647 /// whereas this is O(1). |
| 1648 final int line; |
| 1649 |
| 1650 /// The column on which the key appears. |
| 1651 /// |
| 1652 /// We could get this from [location], but that requires a binary search |
| 1653 /// whereas this is O(1). |
| 1654 final int column; |
| 1655 |
| 1642 /// Whether this key must exist for the document to be scanned. | 1656 /// Whether this key must exist for the document to be scanned. |
| 1643 final bool required; | 1657 final bool required; |
| 1644 | 1658 |
| 1645 _SimpleKey(this.tokenNumber, this.location, {bool required}) | 1659 _SimpleKey(this.tokenNumber, this.line, this.column, this.location, |
| 1660 {bool required}) |
| 1646 : required = required; | 1661 : required = required; |
| 1647 } | 1662 } |
| 1648 | 1663 |
| 1649 /// An enum of chomping indicators that describe how to handle trailing | 1664 /// An enum of chomping indicators that describe how to handle trailing |
| 1650 /// whitespace for a block scalar. | 1665 /// whitespace for a block scalar. |
| 1651 /// | 1666 /// |
| 1652 /// See http://yaml.org/spec/1.2/spec.html#id2794534. | 1667 /// See http://yaml.org/spec/1.2/spec.html#id2794534. |
| 1653 class _Chomping { | 1668 class _Chomping { |
| 1654 /// All trailing whitespace is discarded. | 1669 /// All trailing whitespace is discarded. |
| 1655 static const STRIP = const _Chomping("STRIP"); | 1670 static const STRIP = const _Chomping("STRIP"); |
| 1656 | 1671 |
| 1657 /// A single trailing newline is retained. | 1672 /// A single trailing newline is retained. |
| 1658 static const CLIP = const _Chomping("CLIP"); | 1673 static const CLIP = const _Chomping("CLIP"); |
| 1659 | 1674 |
| 1660 /// All trailing whitespace is preserved. | 1675 /// All trailing whitespace is preserved. |
| 1661 static const KEEP = const _Chomping("KEEP"); | 1676 static const KEEP = const _Chomping("KEEP"); |
| 1662 | 1677 |
| 1663 final String name; | 1678 final String name; |
| 1664 | 1679 |
| 1665 const _Chomping(this.name); | 1680 const _Chomping(this.name); |
| 1666 | 1681 |
| 1667 String toString() => name; | 1682 String toString() => name; |
| 1668 } | 1683 } |
| OLD | NEW |