Chromium Code Reviews| OLD | NEW |
|---|---|
| 1 // Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 library yaml.scanner; | 5 library yaml.scanner; |
| 6 | 6 |
| 7 import 'package:collection/collection.dart'; | 7 import 'package:collection/collection.dart'; |
| 8 import 'package:string_scanner/string_scanner.dart'; | 8 import 'package:string_scanner/string_scanner.dart'; |
| 9 import 'package:source_span/source_span.dart'; | 9 import 'package:source_span/source_span.dart'; |
| 10 | 10 |
| (...skipping 274 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 285 /// If so, this sets the scanner's last match to that indicator. | 285 /// If so, this sets the scanner's last match to that indicator. |
| 286 bool get _isDocumentIndicator { | 286 bool get _isDocumentIndicator { |
| 287 return _scanner.column == 0 && _isBlankOrEndAt(3) && | 287 return _scanner.column == 0 && _isBlankOrEndAt(3) && |
| 288 (_scanner.matches('---') || _scanner.matches('...')); | 288 (_scanner.matches('---') || _scanner.matches('...')); |
| 289 } | 289 } |
| 290 | 290 |
| 291 /// Creates a scanner that scans [source]. | 291 /// Creates a scanner that scans [source]. |
| 292 /// | 292 /// |
| 293 /// [sourceUrl] can be a String or a [Uri]. | 293 /// [sourceUrl] can be a String or a [Uri]. |
| 294 Scanner(String source, {sourceUrl}) | 294 Scanner(String source, {sourceUrl}) |
| 295 : _scanner = new SpanScanner.eager(source, sourceUrl: sourceUrl); | 295 : _scanner = new SpanScanner(source, sourceUrl: sourceUrl); |
|
Bob Nystrom
2015/09/02 23:13:14
Do you need to merge in that previous change?
nweiz
2015/09/02 23:37:59
Oops!
| |
| 296 | 296 |
| 297 /// Consumes and returns the next token. | 297 /// Consumes and returns the next token. |
| 298 Token scan() { | 298 Token scan() { |
| 299 if (_streamEndProduced) throw new StateError("Out of tokens."); | 299 if (_streamEndProduced) throw new StateError("Out of tokens."); |
| 300 if (!_tokenAvailable) _fetchMoreTokens(); | 300 if (!_tokenAvailable) _fetchMoreTokens(); |
| 301 | 301 |
| 302 var token = _tokens.removeFirst(); | 302 var token = _tokens.removeFirst(); |
| 303 _tokenAvailable = false; | 303 _tokenAvailable = false; |
| 304 _tokensParsed++; | 304 _tokensParsed++; |
| 305 _streamEndProduced = token is Token && | 305 _streamEndProduced = token is Token && |
| (...skipping 172 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 478 | 478 |
| 479 // libyaml requires that all simple keys be a single line and no longer | 479 // libyaml requires that all simple keys be a single line and no longer |
| 480 // than 1024 characters. However, in section 7.4.2 of the spec | 480 // than 1024 characters. However, in section 7.4.2 of the spec |
| 481 // (http://yaml.org/spec/1.2/spec.html#id2790832), these restrictions are | 481 // (http://yaml.org/spec/1.2/spec.html#id2790832), these restrictions are |
| 482 // only applied when the curly braces are omitted. It's difficult to | 482 // only applied when the curly braces are omitted. It's difficult to |
| 483 // retain enough context to know which keys need to have the restriction | 483 // retain enough context to know which keys need to have the restriction |
| 484 // placed on them, so for now we go the other direction and allow | 484 // placed on them, so for now we go the other direction and allow |
| 485 // everything but multiline simple keys in a block context. | 485 // everything but multiline simple keys in a block context. |
| 486 if (!_inBlockContext) continue; | 486 if (!_inBlockContext) continue; |
| 487 | 487 |
| 488 if (key.line == _scanner.line) continue; | 488 if (key.location.line == _scanner.line) continue; |
| 489 | 489 |
| 490 if (key.required) { | 490 if (key.required) { |
| 491 throw new YamlException("Expected ':'.", _scanner.emptySpan); | 491 throw new YamlException("Expected ':'.", _scanner.emptySpan); |
| 492 } | 492 } |
| 493 | 493 |
| 494 _simpleKeys[i] = null; | 494 _simpleKeys[i] = null; |
| 495 } | 495 } |
| 496 } | 496 } |
| 497 | 497 |
| 498 /// Checks if a simple key may start at the current position and saves it if | 498 /// Checks if a simple key may start at the current position and saves it if |
| 499 /// so. | 499 /// so. |
| 500 void _saveSimpleKey() { | 500 void _saveSimpleKey() { |
| 501 // A simple key is required at the current position if the scanner is in the | 501 // A simple key is required at the current position if the scanner is in the |
| 502 // block context and the current column coincides with the indentation | 502 // block context and the current column coincides with the indentation |
| 503 // level. | 503 // level. |
| 504 var required = _inBlockContext && _indent == _scanner.column; | 504 var required = _inBlockContext && _indent == _scanner.column; |
| 505 | 505 |
| 506 // A simple key is required only when it is the first token in the current | 506 // A simple key is required only when it is the first token in the current |
| 507 // line. Therefore it is always allowed. But we add a check anyway. | 507 // line. Therefore it is always allowed. But we add a check anyway. |
| 508 assert(_simpleKeyAllowed || !required); | 508 assert(_simpleKeyAllowed || !required); |
| 509 | 509 |
| 510 if (!_simpleKeyAllowed) return; | 510 if (!_simpleKeyAllowed) return; |
| 511 | 511 |
| 512 // If the current position may start a simple key, save it. | 512 // If the current position may start a simple key, save it. |
| 513 _removeSimpleKey(); | 513 _removeSimpleKey(); |
| 514 _simpleKeys[_simpleKeys.length - 1] = new _SimpleKey( | 514 _simpleKeys[_simpleKeys.length - 1] = new _SimpleKey( |
| 515 _tokensParsed + _tokens.length, | 515 _tokensParsed + _tokens.length, |
| 516 _scanner.line, | |
| 517 _scanner.column, | |
| 518 _scanner.location, | 516 _scanner.location, |
| 519 required: required); | 517 required: required); |
| 520 } | 518 } |
| 521 | 519 |
| 522 /// Removes a potential simple key at the current flow level. | 520 /// Removes a potential simple key at the current flow level. |
| 523 void _removeSimpleKey() { | 521 void _removeSimpleKey() { |
| 524 var key = _simpleKeys.last; | 522 var key = _simpleKeys.last; |
| 525 if (key != null && key.required) { | 523 if (key != null && key.required) { |
| 526 throw new YamlException("Could not find expected ':' for simple key.", | 524 throw new YamlException("Could not find expected ':' for simple key.", |
| 527 key.location.pointSpan()); | 525 key.location.pointSpan()); |
| (...skipping 127 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 655 if (_inBlockContext) { | 653 if (_inBlockContext) { |
| 656 if (!_simpleKeyAllowed) { | 654 if (!_simpleKeyAllowed) { |
| 657 throw new YamlException( | 655 throw new YamlException( |
| 658 "Block sequence entries are not allowed here.", | 656 "Block sequence entries are not allowed here.", |
| 659 _scanner.emptySpan); | 657 _scanner.emptySpan); |
| 660 } | 658 } |
| 661 | 659 |
| 662 _rollIndent( | 660 _rollIndent( |
| 663 _scanner.column, | 661 _scanner.column, |
| 664 TokenType.BLOCK_SEQUENCE_START, | 662 TokenType.BLOCK_SEQUENCE_START, |
| 665 _scanner.location); | 663 _scanner.emptySpan.start); |
| 666 } else { | 664 } else { |
| 667 // It is an error for the '-' indicator to occur in the flow context, but | 665 // It is an error for the '-' indicator to occur in the flow context, but |
| 668 // we let the Parser detect and report it because it's able to point to | 666 // we let the Parser detect and report it because it's able to point to |
| 669 // the context. | 667 // the context. |
| 670 } | 668 } |
| 671 | 669 |
| 672 _removeSimpleKey(); | 670 _removeSimpleKey(); |
| 673 _simpleKeyAllowed = true; | 671 _simpleKeyAllowed = true; |
| 674 _addCharToken(TokenType.BLOCK_ENTRY); | 672 _addCharToken(TokenType.BLOCK_ENTRY); |
| 675 } | 673 } |
| 676 | 674 |
| 677 /// Produces the [TokenType.KEY] token. | 675 /// Produces the [TokenType.KEY] token. |
| 678 void _fetchKey() { | 676 void _fetchKey() { |
| 679 if (_inBlockContext) { | 677 if (_inBlockContext) { |
| 680 if (!_simpleKeyAllowed) { | 678 if (!_simpleKeyAllowed) { |
| 681 throw new YamlException("Mapping keys are not allowed here.", | 679 throw new YamlException("Mapping keys are not allowed here.", |
| 682 _scanner.emptySpan); | 680 _scanner.emptySpan); |
| 683 } | 681 } |
| 684 | 682 |
| 685 _rollIndent( | 683 _rollIndent( |
| 686 _scanner.column, | 684 _scanner.column, |
| 687 TokenType.BLOCK_MAPPING_START, | 685 TokenType.BLOCK_MAPPING_START, |
| 688 _scanner.location); | 686 _scanner.emptySpan.start); |
| 689 } | 687 } |
| 690 | 688 |
| 691 // Simple keys are allowed after `?` in a block context. | 689 // Simple keys are allowed after `?` in a block context. |
| 692 _simpleKeyAllowed = _inBlockContext; | 690 _simpleKeyAllowed = _inBlockContext; |
| 693 _addCharToken(TokenType.KEY); | 691 _addCharToken(TokenType.KEY); |
| 694 } | 692 } |
| 695 | 693 |
| 696 /// Produces the [TokenType.VALUE] token. | 694 /// Produces the [TokenType.VALUE] token. |
| 697 void _fetchValue() { | 695 void _fetchValue() { |
| 698 var simpleKey = _simpleKeys.last; | 696 var simpleKey = _simpleKeys.last; |
| 699 if (simpleKey != null) { | 697 if (simpleKey != null) { |
| 700 // Add a [TokenType.KEY] directive before the first token of the simple | 698 // Add a [TokenType.KEY] directive before the first token of the simple |
| 701 // key so the parser knows that it's part of a key/value pair. | 699 // key so the parser knows that it's part of a key/value pair. |
| 702 _tokens.insert(simpleKey.tokenNumber - _tokensParsed, | 700 _tokens.insert(simpleKey.tokenNumber - _tokensParsed, |
| 703 new Token(TokenType.KEY, simpleKey.location.pointSpan())); | 701 new Token(TokenType.KEY, simpleKey.location.pointSpan())); |
| 704 | 702 |
| 705 // In the block context, we may need to add the | 703 // In the block context, we may need to add the |
| 706 // [TokenType.BLOCK_MAPPING_START] token. | 704 // [TokenType.BLOCK_MAPPING_START] token. |
| 707 _rollIndent( | 705 _rollIndent( |
| 708 simpleKey.column, | 706 simpleKey.location.column, |
| 709 TokenType.BLOCK_MAPPING_START, | 707 TokenType.BLOCK_MAPPING_START, |
| 710 simpleKey.location, | 708 simpleKey.location, |
| 711 tokenNumber: simpleKey.tokenNumber); | 709 tokenNumber: simpleKey.tokenNumber); |
| 712 | 710 |
| 713 // Remove the simple key. | 711 // Remove the simple key. |
| 714 _simpleKeys[_simpleKeys.length - 1] = null; | 712 _simpleKeys[_simpleKeys.length - 1] = null; |
| 715 | 713 |
| 716 // A simple key cannot follow another simple key. | 714 // A simple key cannot follow another simple key. |
| 717 _simpleKeyAllowed = false; | 715 _simpleKeyAllowed = false; |
| 718 } else if (_inBlockContext) { | 716 } else if (_inBlockContext) { |
| (...skipping 915 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 1634 /// This is the index relative to all tokens emitted, rather than relative to | 1632 /// This is the index relative to all tokens emitted, rather than relative to |
| 1635 /// [_tokens]. | 1633 /// [_tokens]. |
| 1636 final int tokenNumber; | 1634 final int tokenNumber; |
| 1637 | 1635 |
| 1638 /// The source location of the beginning of the simple key. | 1636 /// The source location of the beginning of the simple key. |
| 1639 /// | 1637 /// |
| 1640 /// This is used for error reporting and for determining when a simple key is | 1638 /// This is used for error reporting and for determining when a simple key is |
| 1641 /// no longer on the current line. | 1639 /// no longer on the current line. |
| 1642 final SourceLocation location; | 1640 final SourceLocation location; |
| 1643 | 1641 |
| 1644 /// The line on which the key appears. | |
| 1645 /// | |
| 1646 /// We could get this from [location], but that requires a binary search | |
| 1647 /// whereas this is O(1). | |
| 1648 final int line; | |
| 1649 | |
| 1650 /// The column on which the key appears. | |
| 1651 /// | |
| 1652 /// We could get this from [location], but that requires a binary search | |
| 1653 /// whereas this is O(1). | |
| 1654 final int column; | |
| 1655 | |
| 1656 /// Whether this key must exist for the document to be scanned. | 1642 /// Whether this key must exist for the document to be scanned. |
| 1657 final bool required; | 1643 final bool required; |
| 1658 | 1644 |
| 1659 _SimpleKey(this.tokenNumber, this.line, this.column, this.location, | 1645 _SimpleKey(this.tokenNumber, this.location, {bool required}) |
| 1660 {bool required}) | |
| 1661 : required = required; | 1646 : required = required; |
| 1662 } | 1647 } |
| 1663 | 1648 |
| 1664 /// An enum of chomping indicators that describe how to handle trailing | 1649 /// An enum of chomping indicators that describe how to handle trailing |
| 1665 /// whitespace for a block scalar. | 1650 /// whitespace for a block scalar. |
| 1666 /// | 1651 /// |
| 1667 /// See http://yaml.org/spec/1.2/spec.html#id2794534. | 1652 /// See http://yaml.org/spec/1.2/spec.html#id2794534. |
| 1668 class _Chomping { | 1653 class _Chomping { |
| 1669 /// All trailing whitespace is discarded. | 1654 /// All trailing whitespace is discarded. |
| 1670 static const STRIP = const _Chomping("STRIP"); | 1655 static const STRIP = const _Chomping("STRIP"); |
| 1671 | 1656 |
| 1672 /// A single trailing newline is retained. | 1657 /// A single trailing newline is retained. |
| 1673 static const CLIP = const _Chomping("CLIP"); | 1658 static const CLIP = const _Chomping("CLIP"); |
| 1674 | 1659 |
| 1675 /// All trailing whitespace is preserved. | 1660 /// All trailing whitespace is preserved. |
| 1676 static const KEEP = const _Chomping("KEEP"); | 1661 static const KEEP = const _Chomping("KEEP"); |
| 1677 | 1662 |
| 1678 final String name; | 1663 final String name; |
| 1679 | 1664 |
| 1680 const _Chomping(this.name); | 1665 const _Chomping(this.name); |
| 1681 | 1666 |
| 1682 String toString() => name; | 1667 String toString() => name; |
| 1683 } | 1668 } |
| OLD | NEW |