Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(47)

Side by Side Diff: dart/sdk/lib/_internal/compiler/implementation/scanner/scanner.dart

Issue 87813002: Fix crashes in scanner. (Closed) Base URL: https://dart.googlecode.com/svn/branches/bleeding_edge
Patch Set: Created 7 years ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
OLDNEW
1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file
2 // for details. All rights reserved. Use of this source code is governed by a 2 // for details. All rights reserved. Use of this source code is governed by a
3 // BSD-style license that can be found in the LICENSE file. 3 // BSD-style license that can be found in the LICENSE file.
4 4
5 part of scanner; 5 part of scanner;
6 6
7 abstract class Scanner { 7 abstract class Scanner {
8 Token tokenize(); 8 Token tokenize();
9 9
10 factory Scanner(SourceFile file, {bool includeComments: false}) { 10 factory Scanner(SourceFile file, {bool includeComments: false}) {
11 if (file is Utf8BytesSourceFile) { 11 if (file is Utf8BytesSourceFile) {
12 return new Utf8BytesScanner(file, includeComments: includeComments); 12 return new Utf8BytesScanner(file, includeComments: includeComments);
13 } else { 13 } else {
14 return new StringScanner(file, includeComments: includeComments); 14 return new StringScanner(file, includeComments: includeComments);
15 } 15 }
16 } 16 }
17 } 17 }
18 18
19 abstract class AbstractScanner implements Scanner { 19 abstract class AbstractScanner implements Scanner {
20 // TODO(ahe): Move this class to implementation.
21
20 final bool includeComments; 22 final bool includeComments;
21 23
22 /** 24 /**
23 * The string offset for the next token that will be created. 25 * The string offset for the next token that will be created.
24 * 26 *
25 * Note that in the [Utf8BytesScanner], [stringOffset] and [scanOffset] values 27 * Note that in the [Utf8BytesScanner], [stringOffset] and [scanOffset] values
26 * are different. One string character can be encoded using multiple UTF-8 28 * are different. One string character can be encoded using multiple UTF-8
27 * bytes. 29 * bytes.
28 */ 30 */
29 int tokenStart = -1; 31 int tokenStart = -1;
(...skipping 149 matching lines...) Expand 10 before | Expand all | Expand 10 after
179 181
180 /** Documentation in subclass [ArrayBasedScanner]. */ 182 /** Documentation in subclass [ArrayBasedScanner]. */
181 void appendGtGt(PrecedenceInfo info); 183 void appendGtGt(PrecedenceInfo info);
182 184
183 /** Documentation in subclass [ArrayBasedScanner]. */ 185 /** Documentation in subclass [ArrayBasedScanner]. */
184 void appendComment(start, bool asciiOnly); 186 void appendComment(start, bool asciiOnly);
185 187
186 /** Documentation in subclass [ArrayBasedScanner]. */ 188 /** Documentation in subclass [ArrayBasedScanner]. */
187 void discardOpenLt(); 189 void discardOpenLt();
188 190
189 // TODO(ahe): Move this class to implementation. 191 /// Return true when at EOF.
192 bool atEndOfFile();
190 193
191 Token tokenize() { 194 Token tokenize() {
192 int next = advance(); 195 while (!atEndOfFile()) {
193 while (!identical(next, $EOF)) { 196 int next = advance();
194 next = bigSwitch(next); 197 while (!identical(next, $EOF)) {
198 next = bigSwitch(next);
199 }
200 if (atEndOfFile()) {
201 appendEofToken();
202 } else {
203 error('blah');
Johnni Winther 2013/11/26 12:11:12 Strange error message!
ahe 2013/11/29 10:48:38 Done.
204 }
195 } 205 }
196 appendEofToken();
197 206
198 if (file != null) { 207 if (file != null) {
199 file.length = stringOffset; 208 file.length = stringOffset;
200 // One additional line start at the end, see [SourceFile.lineStarts]. 209 // One additional line start at the end, see [SourceFile.lineStarts].
201 lineStarts.add(stringOffset + 1); 210 lineStarts.add(stringOffset + 1);
202 file.lineStarts = lineStarts; 211 file.lineStarts = lineStarts;
203 } 212 }
204 213
205 return firstToken(); 214 return firstToken();
206 } 215 }
(...skipping 707 matching lines...) Expand 10 before | Expand all | Expand 10 after
914 } 923 }
915 if (identical(next, $EOF)) return next; 924 if (identical(next, $EOF)) return next;
916 next = advance(); // Move past the $STX. 925 next = advance(); // Move past the $STX.
917 beginToken(); // The string interpolation suffix starts here. 926 beginToken(); // The string interpolation suffix starts here.
918 return next; 927 return next;
919 } 928 }
920 929
921 int tokenizeInterpolatedIdentifier(int next) { 930 int tokenizeInterpolatedIdentifier(int next) {
922 appendPrecedenceToken(STRING_INTERPOLATION_IDENTIFIER_INFO); 931 appendPrecedenceToken(STRING_INTERPOLATION_IDENTIFIER_INFO);
923 beginToken(); // The identifier starts here. 932 beginToken(); // The identifier starts here.
924 next = tokenizeKeywordOrIdentifier(next, false); 933
934 if ($a <= next && next <= $z) {
935 next = tokenizeKeywordOrIdentifier(next, false);
936 } else if (($A <= next && next <= $Z) || identical(next, $_)) {
937 next = tokenizeIdentifier(next, scanOffset, false);
938 } else {
939 error("expected identifier or '{'", shouldAdvance: false);
940 }
925 beginToken(); // The string interpolation suffix starts here. 941 beginToken(); // The string interpolation suffix starts here.
926 return next; 942 return next;
927 } 943 }
928 944
929 int tokenizeSingleLineRawString(int next, int quoteChar, int start) { 945 int tokenizeSingleLineRawString(int next, int quoteChar, int start) {
930 bool asciiOnly = true; 946 bool asciiOnly = true;
931 next = advance(); // Advance past the quote. 947 next = advance(); // Advance past the quote.
932 while (next != $EOF) { 948 while (next != $EOF) {
933 if (identical(next, quoteChar)) { 949 if (identical(next, quoteChar)) {
934 if (!asciiOnly) handleUnicode(start); 950 if (!asciiOnly) handleUnicode(start);
(...skipping 93 matching lines...) Expand 10 before | Expand all | Expand 10 after
1028 } else if (next > 127) { 1044 } else if (next > 127) {
1029 asciiOnlyString = false; 1045 asciiOnlyString = false;
1030 asciiOnlyLine = false; 1046 asciiOnlyLine = false;
1031 } 1047 }
1032 next = advance(); 1048 next = advance();
1033 } 1049 }
1034 if (!asciiOnlyLine) handleUnicode(unicodeStart); 1050 if (!asciiOnlyLine) handleUnicode(unicodeStart);
1035 return error("unterminated string literal"); 1051 return error("unterminated string literal");
1036 } 1052 }
1037 1053
1038 int error(String message) { 1054 int error(String message, {bool shouldAdvance: true}) {
1039 appendStringToken(BAD_INPUT_INFO, message); 1055 appendStringToken(BAD_INPUT_INFO, message);
1040 return advance(); // Ensure progress. 1056 if (atEndOfFile()) return $EOF;
1057 if (shouldAdvance) {
1058 return advance(); // Ensure progress.
1059 } else {
1060 return -1;
1061 }
1041 } 1062 }
1042 1063
1043 void unmatchedBeginGroup(BeginGroupToken begin) { 1064 void unmatchedBeginGroup(BeginGroupToken begin) {
1044 String error = 'unmatched "${begin.stringValue}"'; 1065 String error = 'unmatched "${begin.stringValue}"';
1045 Token close = 1066 Token close =
1046 new StringToken.fromString( 1067 new StringToken.fromString(
1047 BAD_INPUT_INFO, error, begin.charOffset, canonicalize: true); 1068 BAD_INPUT_INFO, error, begin.charOffset, canonicalize: true);
1048 1069
1049 // We want to ensure that unmatched BeginGroupTokens are reported 1070 // We want to ensure that unmatched BeginGroupTokens are reported
1050 // as errors. However, the rest of the parser assumes the groups 1071 // as errors. However, the rest of the parser assumes the groups
1051 // are well-balanced and will never look at the endGroup 1072 // are well-balanced and will never look at the endGroup
1052 // token. This is a nice property that allows us to skip quickly 1073 // token. This is a nice property that allows us to skip quickly
1053 // over correct code. By inserting an additional error token in 1074 // over correct code. By inserting an additional error token in
1054 // the stream, we can keep ignoring endGroup tokens. 1075 // the stream, we can keep ignoring endGroup tokens.
1055 // 1076 //
1056 // [begin] --next--> [tail] 1077 // [begin] --next--> [tail]
1057 // [begin] --endG--> [close] --next--> [next] --next--> [tail] 1078 // [begin] --endG--> [close] --next--> [next] --next--> [tail]
1058 // 1079 //
1059 // This allows the parser to skip from [begin] via endGroup to [close] and 1080 // This allows the parser to skip from [begin] via endGroup to [close] and
1060 // ignore the [close] token (assuming it's correct), then the error will be 1081 // ignore the [close] token (assuming it's correct), then the error will be
1061 // reported when parsing the [next] token. 1082 // reported when parsing the [next] token.
1062 1083
1063 Token next = new StringToken.fromString( 1084 Token next = new StringToken.fromString(
1064 BAD_INPUT_INFO, error, begin.charOffset, canonicalize: true); 1085 BAD_INPUT_INFO, error, begin.charOffset, canonicalize: true);
1065 begin.endGroup = close; 1086 begin.endGroup = close;
1066 close.next = next; 1087 close.next = next;
1067 next.next = begin.next; 1088 next.next = begin.next;
1068 } 1089 }
1069 } 1090 }
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698