Use SpanScanner to emit better YAML parse errors.

pkg/yaml/lib/src/parser.dart

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 library yaml.parser;
 
 import 'dart:collection';
 
+import 'package:string_scanner/string_scanner.dart';
+
 import 'model.dart';
 import 'yaml_exception.dart';
 import 'yaml_map.dart';
 
 /// Translates a string of characters into a YAML serialization tree.
 ///
 /// This parser is designed to closely follow the spec. All productions in the
 /// spec are numbered, and the corresponding methods in the parser have the same
 /// numbers. This is certainly not the most efficient way of parsing YAML, but
 /// it is the easiest to write and read in the context of the spec.
@@ skipping 90 matching lines @@
   static const BLOCK_IN = 1;
   static const FLOW_OUT = 2;
   static const FLOW_IN = 3;
   static const BLOCK_KEY = 4;
   static const FLOW_KEY = 5;
 
   static const CHOMPING_STRIP = 0;
   static const CHOMPING_KEEP = 1;
   static const CHOMPING_CLIP = 2;
 
-  /// The source string being parsed.
-  final String _s;
-
-  /// The current position in the source string.
-  int _pos = 0;
-
-  /// The length of the string being parsed.
-  final int _len;
-
-  /// The current (0-based) line in the source string.
-  int _line = 0;
-
-  /// The current (0-based) column in the source string.
-  int _column = 0;
+  /// The scanner that's used to scan through the document.
+  final SpanScanner _scanner;
 
   /// Whether we're parsing a bare document (that is, one that doesn't begin
   /// with `---`). Bare documents don't allow `%` immediately following
   /// newlines.
   bool _inBareDocument = false;
 
-  /// The line number of the farthest position that has been parsed successfully
-  /// before backtracking. Used for error reporting.
-  int _farthestLine = 0;
-
-  /// The column number of the farthest position that has been parsed
-  /// successfully before backtracking. Used for error reporting.
-  int _farthestColumn = 0;
-
-  /// The farthest position in the source string that has been parsed
-  /// successfully before backtracking. Used for error reporting.
-  int _farthestPos = 0;
+  /// The state of the scanner when it was the farthest in the document it's
+  /// been.
+  LineScannerState _farthestState;
 
   /// The name of the context of the farthest position that has been parsed
   /// successfully before backtracking. Used for error reporting.
   String _farthestContext = "document";
 
   /// A stack of the names of parse contexts. Used for error reporting.
-  List<String> _contextStack;
+  final _contextStack = <String>["document"];
 
   /// Annotations attached to ranges of the source string that add extra
   /// information to any errors that occur in the annotated range.
-  _RangeMap<String> _errorAnnotations;
+  final _errorAnnotations = new _RangeMap<String>();
 
   /// The buffer containing the string currently being captured.
   StringBuffer _capturedString;
 
   /// The beginning of the current section of the captured string.
   int _captureStart;
 
   /// Whether the current string capture is being overridden.
   bool _capturingAs = false;
 
-  Parser(String s)
-    : this._s = s,
-      _len = s.length,
-      _contextStack = <String>["document"],
-      _errorAnnotations = new _RangeMap();
+  Parser(String yaml, String sourceName)
+      : _scanner = new SpanScanner(yaml, sourceName) {
+    _farthestState = _scanner.state;
+  }
 
-  /// Return the character at the current position, then move that position
-  /// forward one character. Also updates the current line and column numbers.
-  int next() {
-    if (_pos == _len) return -1;
-    var char = _s.codeUnitAt(_pos++);
-    if (isBreak(char)) {
-      _line++;
-      _column = 0;
-    } else {
-      _column++;
-    }
-
-    if (_farthestLine < _line) {
-      _farthestLine = _line;
-      _farthestColumn = _column;
-      _farthestContext = _contextStack.last;
-    } else if (_farthestLine == _line && _farthestColumn < _column) {
-      _farthestColumn = _column;
-      _farthestContext = _contextStack.last;
-    }
-    _farthestPos = _pos;
-
-    return char;
-  }
+  /// Returns the character at the current position, then moves that position
+  /// forward one character.
+  int next() => _scanner.readChar();
 
   /// Returns the code unit at the current position, or the character [i]
   /// characters after the current position.
-  ///
-  /// Returns -1 if this would return a character after the end or before the
-  /// beginning of the input string.
-  int peek([int i = 0]) {
-    var peekPos = _pos + i;
-    return (peekPos >= _len || peekPos < 0) ? -1 : _s.codeUnitAt(peekPos);
-  }
+  int peek([int i = 0]) => _scanner.peekChar(i);
 
   /// The truthiness operator. Returns `false` if [obj] is `null` or `false`,
   /// `true` otherwise.
   bool truth(obj) => obj != null && obj != false;
 
   /// Consumes the current character if it matches [matcher]. Returns the result
   /// of [matcher].
   bool consume(bool matcher(int)) {
     if (matcher(peek())) {
       next();
@@ skipping 21 matching lines @@
     return null; // Unreachable.
   }
 
   /// Calls [consumer] until it returns a falsey value. Returns a list of all
   /// truthy return values of [consumer], or the empty list if it didn't consume
   /// anything.
   ///
   /// Conceptually, repeats a production any number of times.
   List zeroOrMore(consumer()) {
     var out = [];
-    var oldPos = _pos;
+    var oldPos = _scanner.position;
     while (true) {
       var el = consumer();
-      if (!truth(el) || oldPos == _pos) return out;
-      oldPos = _pos;
+      if (!truth(el) || oldPos == _scanner.position) return out;
+      oldPos = _scanner.position;
       out.add(el);
     }
     return null; // Unreachable.
   }
 
   /// Just calls [consumer] and returns its result. Used to make it explicit
   /// that a production is intended to be optional.
   zeroOrOne(consumer()) => consumer();
 
   /// Calls each function in [consumers] until one returns a truthy value, then
   /// returns that.
   or(List<Function> consumers) {
     for (var c in consumers) {
       var res = c();
       if (truth(res)) return res;
     }
     return null;
   }
 
   /// Calls [consumer] and returns its result, but rolls back the parser state
   /// if [consumer] returns a falsey value.
   transaction(consumer()) {
-    var oldPos = _pos;
-    var oldLine = _line;
-    var oldColumn = _column;
+    var oldState = _scanner.state;
     var oldCaptureStart = _captureStart;
     String capturedSoFar = _capturedString == null ? null :
       _capturedString.toString();
     var res = consumer();
+    _refreshFarthestState();
     if (truth(res)) return res;
 
-    _pos = oldPos;
-    _line = oldLine;
-    _column = oldColumn;
+    _scanner.state = oldState;
     _captureStart = oldCaptureStart;
     _capturedString = capturedSoFar == null ? null :
       new StringBuffer(capturedSoFar);
     return res;
   }
 
   /// Consumes [n] characters matching [matcher], or none if there isn't a
   /// complete match. The first argument to [matcher] is the character code, the
   /// second is the index (from 0 to [n] - 1).
   ///
@@ skipping 16 matching lines @@
   String stringOf(bool matcher(int)) =>
     captureString(() => oneOrMore(() => consume(matcher)));
 
   /// Calls [consumer] and returns the string that was consumed while doing so,
   /// or null if [consumer] returned a falsey value. Automatically wraps
   /// [consumer] in `transaction`.
   String captureString(consumer()) {
     // captureString calls may not be nested
     assert(_capturedString == null);
 
-    _captureStart = _pos;
+    _captureStart = _scanner.position;
     _capturedString = new StringBuffer();
     var res = transaction(consumer);
     if (!truth(res)) {
       _captureStart = null;
       _capturedString = null;
       return null;
     }
 
     flushCapture();
     var result = _capturedString.toString();
     _captureStart = null;
     _capturedString = null;
     return result;
   }
 
   captureAs(String replacement, consumer()) =>
       captureAndTransform(consumer, (_) => replacement);
 
   captureAndTransform(consumer(), String transformation(String captured)) {
     if (_capturedString == null) return consumer();
     if (_capturingAs) return consumer();
 
     flushCapture();
     _capturingAs = true;
     var res = consumer();
     _capturingAs = false;
     if (!truth(res)) return res;
 
-    _capturedString.write(transformation(_s.substring(_captureStart, _pos)));
-    _captureStart = _pos;
+    _capturedString.write(transformation(
+        _scanner.string.substring(_captureStart, _scanner.position)));
+    _captureStart = _scanner.position;
     return res;
   }
 
   void flushCapture() {
-    _capturedString.write(_s.substring(_captureStart, _pos));
-    _captureStart = _pos;
+    _capturedString.write(_scanner.string.substring(
+        _captureStart, _scanner.position));
+    _captureStart = _scanner.position;
   }
 
   /// Adds a tag and an anchor to [node], if they're defined.
   Node addProps(Node node, _Pair<Tag, String> props) {
     if (props == null || node == null) return node;
     if (truth(props.first)) node.tag = props.first;
     if (truth(props.last)) node.anchor = props.last;
     return node;
   }
 
@@ skipping 12 matching lines @@
     } finally {
       var popped = _contextStack.removeLast();
       assert(popped == name);
     }
   }
 
   /// Adds [message] as extra information to any errors that occur between the
   /// current position and the position of the cursor after running [fn]. The
   /// cursor is reset after [fn] is run.
   annotateError(String message, fn()) {
-    var start = _pos;
+    var start = _scanner.position;
     var end;
     transaction(() {
       fn();
-      end = _pos;
+      end = _scanner.position;
       return false;
     });
     _errorAnnotations[new _Range(start, end)] = message;
   }
 
   /// Throws an error with additional context information.
-  error(String message) {
-    // Line and column should be one-based.
-    throw new SyntaxError(_line + 1, _column + 1,
-        "$message (in $_farthestContext).");
-  }
+  void error(String message) =>
+      _scanner.error("$message (in $_farthestContext).");
 
   /// If [result] is falsey, throws an error saying that [expected] was
   /// expected.
   expect(result, String expected) {
     if (truth(result)) return result;
     error("Expected $expected");
   }
 
-  /// Throws an error saying that the parse failed. Uses [_farthestLine],
-  /// [_farthestColumn], and [_farthestContext] to provide additional
+  /// Throws an error saying that the parse failed.
+  ///
+  /// Uses [_farthestState] and [_farthestContext] to provide additional
   /// information.
   parseFailed() {
     var message = "Invalid YAML in $_farthestContext";
-    var extraError = _errorAnnotations[_farthestPos];
+    _refreshFarthestState();
+    _scanner.state = _farthestState;
+
+    var extraError = _errorAnnotations[_scanner.position];
     if (extraError != null) message = "$message ($extraError)";
-    throw new SyntaxError(_farthestLine + 1, _farthestColumn + 1, "$message.");
+    _scanner.error("$message.");
+  }
+
+  /// Update [_farthestState] if the scanner is farther than it's been before.
+  void _refreshFarthestState() {
+    if (_scanner.position <= _farthestState.position) return;
+    _farthestState = _scanner.state;
   }
 
   /// Returns the number of spaces after the current position.
   int countIndentation() {
     var i = 0;
     while (peek(i) == SP) i++;
     return i;
   }
 
   /// Returns the indentation for a block scalar.
   int blockScalarAdditionalIndentation(_BlockHeader header, int indent) {
     if (!header.autoDetectIndent) return header.additionalIndent;
 
     var maxSpaces = 0;
-    var maxSpacesLine = 0;
     var spaces = 0;
     transaction(() {
       do {
         spaces = captureString(() => zeroOrMore(() => consumeChar(SP))).length;
-        if (spaces > maxSpaces) {
-          maxSpaces = spaces;
-          maxSpacesLine = _line;
-        }
+        if (spaces > maxSpaces) maxSpaces = spaces;
       } while (b_break());
       return false;
     });
 
     // If the next non-empty line isn't indented further than the start of the
     // block scalar, that means the scalar is going to be empty. Returning any
     // value > 0 will cause the parser not to consume any text.
     if (spaces <= indent) return 1;
 
     // It's an error for a leading empty line to be indented more than the first
     // non-empty line.
     if (maxSpaces > spaces) {
-      throw new SyntaxError(maxSpacesLine + 1, maxSpaces,
-          "Leading empty lines may not be indented more than the first "
-          "non-empty line.");
+      _scanner.error("Leading empty lines may not be indented more than the "
+          "first non-empty line.");
     }
 
     return spaces - indent;
   }
 
   /// Returns whether the current position is at the beginning of a line.
-  bool get atStartOfLine => _column == 0;
-
-  /// Returns whether the current position is at the end of the input.
-  bool get atEndOfFile => _pos == _len;
+  bool get atStartOfLine => _scanner.column == 0;
 
   /// Given an indicator character, returns the type of that indicator (or null
   /// if the indicator isn't found.
   int indicatorType(int char) {
     switch (char) {
     case HYPHEN: return C_SEQUENCE_ENTRY;
     case QUESTION_MARK: return C_MAPPING_KEY;
     case COLON: return C_MAPPING_VALUE;
     case COMMA: return C_COLLECT_ENTRY;
     case LEFT_BRACKET: return C_SEQUENCE_START;
@@ skipping 11 matching lines @@
     case PERCENT: return C_DIRECTIVE;
     case AT:
     case GRAVE_ACCENT:
       return C_RESERVED;
     default: return null;
     }
   }
 
   // 1
   bool isPrintable(int char) {
+    if (char == null) return false;
     return char == TAB ||
       char == LF ||
       char == CR ||
       (char >= SP && char <= TILDE) ||
       char == NEL ||
       (char >= 0xA0 && char <= 0xD7FF) ||
       (char >= 0xE000 && char <= 0xFFFD) ||
       (char >= 0x10000 && char <= 0x10FFFF);
   }
 
   // 2
-  bool isJson(int char) => char == TAB || (char >= SP && char <= 0x10FFFF);
+  bool isJson(int char) => char != null &&
+      (char == TAB || (char >= SP && char <= 0x10FFFF));
 
   // 22
   bool c_indicator(int type) => consume((c) => indicatorType(c) == type);
 
   // 23
   bool isFlowIndicator(int char) {
     var indicator = indicatorType(char);
     return indicator == C_COLLECT_ENTRY ||
       indicator == C_SEQUENCE_START ||
       indicator == C_SEQUENCE_END ||
@@ skipping 22 matching lines @@
   // 30
   bool b_nonContent() => captureAs("", () => b_break());
 
   // 33
   bool isSpace(int char) => char == SP || char == TAB;
 
   // 34
   bool isNonSpace(int char) => isNonBreak(char) && !isSpace(char);
 
   // 35
-  bool isDecDigit(int char) => char >= NUMBER_0 && char <= NUMBER_9;
+  bool isDecDigit(int char) => char != null && char >= NUMBER_0 &&
+      char <= NUMBER_9;
 
   // 36
   bool isHexDigit(int char) {
+    if (char == null) return false;
     return isDecDigit(char) ||
       (char >= LETTER_A && char <= LETTER_F) ||
       (char >= LETTER_CAP_A && char <= LETTER_CAP_F);
   }
 
   // 41
   bool c_escape() => captureAs("", () => consumeChar(BACKSLASH));
 
   // 42
   bool ns_escNull() => captureAs("\x00", () => consumeChar(NUMBER_0));
@@ skipping 172 matching lines @@
   });
 
   // 75
   bool c_nb_commentText() {
     if (!truth(c_indicator(C_COMMENT))) return false;
     zeroOrMore(() => consume(isNonBreak));
     return true;
   }
 
   // 76
-  bool b_comment() => atEndOfFile || b_nonContent();
+  bool b_comment() => _scanner.isDone || b_nonContent();
 
   // 77
   bool s_b_comment() {
     if (truth(s_separateInLine())) {
       zeroOrOne(c_nb_commentText);
     }
     return b_comment();
   }
 
   // 78
@@ skipping 653 matching lines @@
     case PLUS:
       next();
       return CHOMPING_KEEP;
     default:
       return CHOMPING_CLIP;
     }
   }
 
   // 165
   bool b_chompedLast(int chomping) {
-    if (atEndOfFile) return true;
+    if (_scanner.isDone) return true;
     switch (chomping) {
     case CHOMPING_STRIP:
       return b_nonContent();
     case CHOMPING_CLIP:
     case CHOMPING_KEEP:
       return b_asLineFeed();
     }
     throw "unreachable";
   }
 
@@ skipping 361 matching lines @@
     return s_l_comments();
   });
 
   // 206
   bool c_forbidden() {
     if (!_inBareDocument || !atStartOfLine) return false;
     var forbidden = false;
     transaction(() {
       if (!truth(or([c_directivesEnd, c_documentEnd]))) return;
       var char = peek();
-      forbidden = isBreak(char) || isSpace(char) || atEndOfFile;
+      forbidden = isBreak(char) || isSpace(char) || _scanner.isDone;
       return;
     });
     return forbidden;
   }
 
   // 207
   Node l_bareDocument() {
     try {
       _inBareDocument = true;
       return s_l_blockNode(-1, BLOCK_IN);
@@ skipping 40 matching lines @@
         zeroOrMore(l_documentPrefix);
         doc = zeroOrOne(l_anyDocument);
       } else {
         zeroOrMore(l_documentPrefix);
         doc = zeroOrOne(l_explicitDocument);
       }
       if (truth(doc)) docs.add(doc);
       return doc;
     });
 
-    if (!atEndOfFile) parseFailed();
+    if (!_scanner.isDone) parseFailed();
     return docs;
   }
 }
 
-class SyntaxError extends YamlException {
-  final int _line;
-  final int _column;
-
-  SyntaxError(this._line, this._column, String msg) : super(msg);
-
-  String toString() => "Syntax error on line $_line, column $_column: "
-      "${super.toString()}";
-}
-
 /// A pair of values.
 class _Pair<E, F> {
   E first;
   F last;
 
   _Pair(this.first, this.last);
 
   String toString() => '($first, $last)';
 }
 
@@ skipping 40 matching lines @@
     for (var pair in _contents.reversed) {
       if (pair.first.contains(pos)) return pair.last;
     }
     return null;
   }
 
   /// Associates [value] with [range].
   operator[]=(_Range range, E value) =>
     _contents.add(new _Pair<_Range, E>(range, value));
 }