Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(878)

Side by Side Diff: pkg/yaml/lib/src/scanner.dart

Issue 797103002: Don't include extra whitespace in YAML scalar spans. (Closed) Base URL: https://dart.googlecode.com/svn/branches/bleeding_edge/dart
Patch Set: Created 6 years ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
« no previous file with comments | « pkg/yaml/CHANGELOG.md ('k') | pkg/yaml/pubspec.yaml » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 // Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file 1 // Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file
2 // for details. All rights reserved. Use of this source code is governed by a 2 // for details. All rights reserved. Use of this source code is governed by a
3 // BSD-style license that can be found in the LICENSE file. 3 // BSD-style license that can be found in the LICENSE file.
4 4
5 library yaml.scanner; 5 library yaml.scanner;
6 6
7 import 'package:collection/collection.dart'; 7 import 'package:collection/collection.dart';
8 import 'package:string_scanner/string_scanner.dart'; 8 import 'package:string_scanner/string_scanner.dart';
9 import 'package:source_span/source_span.dart'; 9 import 'package:source_span/source_span.dart';
10 10
(...skipping 1066 matching lines...) Expand 10 before | Expand all | Expand 10 after
1077 _scanner.readChar(); 1077 _scanner.readChar();
1078 char = _scanner.peekChar(); 1078 char = _scanner.peekChar();
1079 } 1079 }
1080 1080
1081 // libyaml manually decodes the URL, but we don't have to do that. 1081 // libyaml manually decodes the URL, but we don't have to do that.
1082 return Uri.decodeFull(_scanner.substring(start)); 1082 return Uri.decodeFull(_scanner.substring(start));
1083 } 1083 }
1084 1084
1085 /// Scans a block scalar. 1085 /// Scans a block scalar.
1086 Token _scanBlockScalar({bool literal: false}) { 1086 Token _scanBlockScalar({bool literal: false}) {
1087 print("scanning block scalar");
Bob Nystrom 2014/12/12 01:47:37 Delete.
1087 var start = _scanner.state; 1088 var start = _scanner.state;
1088 1089
1089 // Eat the indicator '|' or '>'. 1090 // Eat the indicator '|' or '>'.
1090 _scanner.readChar(); 1091 _scanner.readChar();
1091 1092
1092 // Check for a chomping indicator. 1093 // Check for a chomping indicator.
1093 var chomping = _Chomping.CLIP; 1094 var chomping = _Chomping.CLIP;
1094 var increment = 0; 1095 var increment = 0;
1095 var char = _scanner.peekChar(); 1096 var char = _scanner.peekChar();
1096 if (char == PLUS || char == HYPHEN) { 1097 if (char == PLUS || char == HYPHEN) {
(...skipping 52 matching lines...) Expand 10 before | Expand all | Expand 10 after
1149 // needed. 1150 // needed.
1150 var pair = _scanBlockScalarBreaks(indent); 1151 var pair = _scanBlockScalarBreaks(indent);
1151 indent = pair.first; 1152 indent = pair.first;
1152 var trailingBreaks = pair.last; 1153 var trailingBreaks = pair.last;
1153 1154
1154 // Scan the block scalar contents. 1155 // Scan the block scalar contents.
1155 var buffer = new StringBuffer(); 1156 var buffer = new StringBuffer();
1156 var leadingBreak = ''; 1157 var leadingBreak = '';
1157 var leadingBlank = false; 1158 var leadingBlank = false;
1158 var trailingBlank = false; 1159 var trailingBlank = false;
1160 var end = _scanner.position;
1159 while (_scanner.column == indent && !_scanner.isDone) { 1161 while (_scanner.column == indent && !_scanner.isDone) {
1160 // Check for a document indicator. libyaml doesn't do this, but the spec 1162 // Check for a document indicator. libyaml doesn't do this, but the spec
1161 // mandates it. See example 9.5: 1163 // mandates it. See example 9.5:
1162 // http://yaml.org/spec/1.2/spec.html#id2801606. 1164 // http://yaml.org/spec/1.2/spec.html#id2801606.
1163 if (_isDocumentIndicator) break; 1165 if (_isDocumentIndicator) break;
1164 1166
1165 // We are at the beginning of a non-empty line. 1167 // We are at the beginning of a non-empty line.
1166 1168
1167 // Is there trailing whitespace? 1169 // Is there trailing whitespace?
1168 trailingBlank = _isBlank; 1170 trailingBlank = _isBlank;
(...skipping 12 matching lines...) Expand all
1181 buffer.write(trailingBreaks); 1183 buffer.write(trailingBreaks);
1182 1184
1183 // Is there leading whitespace? 1185 // Is there leading whitespace?
1184 leadingBlank = _isBlank; 1186 leadingBlank = _isBlank;
1185 1187
1186 var startPosition = _scanner.position; 1188 var startPosition = _scanner.position;
1187 while (!_isBreakOrEnd) { 1189 while (!_isBreakOrEnd) {
1188 _scanner.readChar(); 1190 _scanner.readChar();
1189 } 1191 }
1190 buffer.write(_scanner.substring(startPosition)); 1192 buffer.write(_scanner.substring(startPosition));
1193 end = _scanner.position;
1191 1194
1192 // libyaml always reads a line here, but this breaks on block scalars at 1195 // libyaml always reads a line here, but this breaks on block scalars at
1193 // the end of the document that end without newlines. See example 8.1: 1196 // the end of the document that end without newlines. See example 8.1:
1194 // http://yaml.org/spec/1.2/spec.html#id2793888. 1197 // http://yaml.org/spec/1.2/spec.html#id2793888.
1195 if (!_scanner.isDone) leadingBreak = _readLine(); 1198 if (!_scanner.isDone) leadingBreak = _readLine();
1196 1199
1197 // Eat the following indentation and spaces. 1200 // Eat the following indentation and spaces.
1198 var pair = _scanBlockScalarBreaks(indent); 1201 var pair = _scanBlockScalarBreaks(indent);
1199 indent = pair.first; 1202 indent = pair.first;
1200 trailingBreaks = pair.last; 1203 trailingBreaks = pair.last;
1201 } 1204 }
1202 1205
1203 // Chomp the tail. 1206 // Chomp the tail.
1204 if (chomping != _Chomping.STRIP) buffer.write(leadingBreak); 1207 if (chomping != _Chomping.STRIP) buffer.write(leadingBreak);
1205 if (chomping == _Chomping.KEEP) buffer.write(trailingBreaks); 1208 if (chomping == _Chomping.KEEP) buffer.write(trailingBreaks);
1206 1209
1207 return new ScalarToken(_scanner.spanFrom(start), buffer.toString(), 1210 return new ScalarToken(_scanner.spanFrom(start, end), buffer.toString(),
1208 literal ? ScalarStyle.LITERAL : ScalarStyle.FOLDED); 1211 literal ? ScalarStyle.LITERAL : ScalarStyle.FOLDED);
1209 } 1212 }
1210 1213
1211 /// Scans indentation spaces and line breaks for a block scalar. 1214 /// Scans indentation spaces and line breaks for a block scalar.
1212 /// 1215 ///
1213 /// Determines the intendation level if needed. Returns the new indentation 1216 /// Determines the intendation level if needed. Returns the new indentation
1214 /// level and the text of the line breaks. 1217 /// level and the text of the line breaks.
1215 Pair<int, String> _scanBlockScalarBreaks(int indent) { 1218 Pair<int, String> _scanBlockScalarBreaks(int indent) {
1216 var maxIndent = 0; 1219 var maxIndent = 0;
1217 var breaks = new StringBuffer(); 1220 var breaks = new StringBuffer();
(...skipping 205 matching lines...) Expand 10 before | Expand all | Expand 10 after
1423 // Eat the right quote. 1426 // Eat the right quote.
1424 _scanner.readChar(); 1427 _scanner.readChar();
1425 1428
1426 return new ScalarToken(_scanner.spanFrom(start), buffer.toString(), 1429 return new ScalarToken(_scanner.spanFrom(start), buffer.toString(),
1427 singleQuote ? ScalarStyle.SINGLE_QUOTED : ScalarStyle.DOUBLE_QUOTED); 1430 singleQuote ? ScalarStyle.SINGLE_QUOTED : ScalarStyle.DOUBLE_QUOTED);
1428 } 1431 }
1429 1432
1430 /// Scans a plain scalar. 1433 /// Scans a plain scalar.
1431 Token _scanPlainScalar() { 1434 Token _scanPlainScalar() {
1432 var start = _scanner.state; 1435 var start = _scanner.state;
1436 var end = _scanner.state;
1433 var buffer = new StringBuffer(); 1437 var buffer = new StringBuffer();
1434 var leadingBreak = ''; 1438 var leadingBreak = '';
1435 var trailingBreaks = ''; 1439 var trailingBreaks = '';
1436 var whitespace = new StringBuffer(); 1440 var whitespace = new StringBuffer();
1437 var indent = _indent + 1; 1441 var indent = _indent + 1;
1438 1442
1439 while (true) { 1443 while (true) {
1440 // Check for a document indicator. 1444 // Check for a document indicator.
1441 if (_isDocumentIndicator) break; 1445 if (_isDocumentIndicator) break;
1442 1446
(...skipping 16 matching lines...) Expand all
1459 } 1463 }
1460 } 1464 }
1461 1465
1462 // libyaml's notion of valid identifiers differs substantially from YAML 1466 // libyaml's notion of valid identifiers differs substantially from YAML
1463 // 1.2's. We use [_isPlainChar] instead of libyaml's character here. 1467 // 1.2's. We use [_isPlainChar] instead of libyaml's character here.
1464 var startPosition = _scanner.position; 1468 var startPosition = _scanner.position;
1465 while (_isPlainChar) { 1469 while (_isPlainChar) {
1466 _scanner.readChar(); 1470 _scanner.readChar();
1467 } 1471 }
1468 buffer.write(_scanner.substring(startPosition)); 1472 buffer.write(_scanner.substring(startPosition));
1473 end = _scanner.state;
1469 1474
1470 // Is it the end? 1475 // Is it the end?
1471 if (!_isBlank && !_isBreak) break; 1476 if (!_isBlank && !_isBreak) break;
1472 1477
1473 while (_isBlank || _isBreak) { 1478 while (_isBlank || _isBreak) {
1474 if (_isBlank) { 1479 if (_isBlank) {
1475 // Check for a tab character messing up the intendation. 1480 // Check for a tab character messing up the intendation.
1476 if (leadingBreak.isNotEmpty && _scanner.column < indent && 1481 if (leadingBreak.isNotEmpty && _scanner.column < indent &&
1477 _scanner.peekChar() == TAB) { 1482 _scanner.peekChar() == TAB) {
1478 _scanner.error("Expected a space but found a tab.", length: 1); 1483 _scanner.error("Expected a space but found a tab.", length: 1);
(...skipping 15 matching lines...) Expand all
1494 } 1499 }
1495 } 1500 }
1496 1501
1497 // Check the indentation level. 1502 // Check the indentation level.
1498 if (_inBlockContext && _scanner.column < indent) break; 1503 if (_inBlockContext && _scanner.column < indent) break;
1499 } 1504 }
1500 1505
1501 // Allow a simple key after a plain scalar with leading blanks. 1506 // Allow a simple key after a plain scalar with leading blanks.
1502 if (leadingBreak.isNotEmpty) _simpleKeyAllowed = true; 1507 if (leadingBreak.isNotEmpty) _simpleKeyAllowed = true;
1503 1508
1504 return new ScalarToken(_scanner.spanFrom(start), buffer.toString(), 1509 return new ScalarToken(_scanner.spanFrom(start, end), buffer.toString(),
1505 ScalarStyle.PLAIN); 1510 ScalarStyle.PLAIN);
1506 } 1511 }
1507 1512
1508 /// Moves past the current line break, if there is one. 1513 /// Moves past the current line break, if there is one.
1509 void _skipLine() { 1514 void _skipLine() {
1510 var char = _scanner.peekChar(); 1515 var char = _scanner.peekChar();
1511 if (char != CR && char != LF) return; 1516 if (char != CR && char != LF) return;
1512 _scanner.readChar(); 1517 _scanner.readChar();
1513 if (char == CR && _scanner.peekChar() == LF) _scanner.readChar(); 1518 if (char == CR && _scanner.peekChar() == LF) _scanner.readChar();
1514 } 1519 }
(...skipping 139 matching lines...) Expand 10 before | Expand all | Expand 10 after
1654 1659
1655 /// All trailing whitespace is preserved. 1660 /// All trailing whitespace is preserved.
1656 static const KEEP = const _Chomping("KEEP"); 1661 static const KEEP = const _Chomping("KEEP");
1657 1662
1658 final String name; 1663 final String name;
1659 1664
1660 const _Chomping(this.name); 1665 const _Chomping(this.name);
1661 1666
1662 String toString() => name; 1667 String toString() => name;
1663 } 1668 }
OLDNEW
« no previous file with comments | « pkg/yaml/CHANGELOG.md ('k') | pkg/yaml/pubspec.yaml » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698