Chromium Code Reviews| OLD | NEW |
|---|---|
| 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 part of scanner; | 5 part of scanner; |
| 6 | 6 |
| 7 abstract class Scanner { | 7 abstract class Scanner { |
| 8 Token tokenize(); | 8 Token tokenize(); |
| 9 | 9 |
| 10 factory Scanner(SourceFile file, {bool includeComments: false}) { | 10 factory Scanner(SourceFile file, {bool includeComments: false}) { |
| 11 if (file is Utf8BytesSourceFile) { | 11 if (file is Utf8BytesSourceFile) { |
| 12 return new Utf8BytesScanner(file, includeComments: includeComments); | 12 return new Utf8BytesScanner(file, includeComments: includeComments); |
| 13 } else { | 13 } else { |
| 14 return new StringScanner(file, includeComments: includeComments); | 14 return new StringScanner(file, includeComments: includeComments); |
| 15 } | 15 } |
| 16 } | 16 } |
| 17 } | 17 } |
| 18 | 18 |
| 19 abstract class AbstractScanner implements Scanner { | 19 abstract class AbstractScanner implements Scanner { |
| 20 // TODO(ahe): Move this class to implementation. | |
| 21 | |
| 20 final bool includeComments; | 22 final bool includeComments; |
| 21 | 23 |
| 22 /** | 24 /** |
| 23 * The string offset for the next token that will be created. | 25 * The string offset for the next token that will be created. |
| 24 * | 26 * |
| 25 * Note that in the [Utf8BytesScanner], [stringOffset] and [scanOffset] values | 27 * Note that in the [Utf8BytesScanner], [stringOffset] and [scanOffset] values |
| 26 * are different. One string character can be encoded using multiple UTF-8 | 28 * are different. One string character can be encoded using multiple UTF-8 |
| 27 * bytes. | 29 * bytes. |
| 28 */ | 30 */ |
| 29 int tokenStart = -1; | 31 int tokenStart = -1; |
| (...skipping 149 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 179 | 181 |
| 180 /** Documentation in subclass [ArrayBasedScanner]. */ | 182 /** Documentation in subclass [ArrayBasedScanner]. */ |
| 181 void appendGtGt(PrecedenceInfo info); | 183 void appendGtGt(PrecedenceInfo info); |
| 182 | 184 |
| 183 /** Documentation in subclass [ArrayBasedScanner]. */ | 185 /** Documentation in subclass [ArrayBasedScanner]. */ |
| 184 void appendComment(start, bool asciiOnly); | 186 void appendComment(start, bool asciiOnly); |
| 185 | 187 |
| 186 /** Documentation in subclass [ArrayBasedScanner]. */ | 188 /** Documentation in subclass [ArrayBasedScanner]. */ |
| 187 void discardOpenLt(); | 189 void discardOpenLt(); |
| 188 | 190 |
| 189 // TODO(ahe): Move this class to implementation. | 191 /// Return true when at EOF. |
| 192 bool atEndOfFile(); | |
| 190 | 193 |
| 191 Token tokenize() { | 194 Token tokenize() { |
| 192 int next = advance(); | 195 while (!atEndOfFile()) { |
| 193 while (!identical(next, $EOF)) { | 196 int next = advance(); |
| 194 next = bigSwitch(next); | 197 while (!identical(next, $EOF)) { |
| 198 next = bigSwitch(next); | |
| 199 } | |
| 200 if (atEndOfFile()) { | |
| 201 appendEofToken(); | |
| 202 } else { | |
| 203 error('blah'); | |
|
Johnni Winther
2013/11/26 12:11:12
Strange error message!
ahe
2013/11/29 10:48:38
Done.
| |
| 204 } | |
| 195 } | 205 } |
| 196 appendEofToken(); | |
| 197 | 206 |
| 198 if (file != null) { | 207 if (file != null) { |
| 199 file.length = stringOffset; | 208 file.length = stringOffset; |
| 200 // One additional line start at the end, see [SourceFile.lineStarts]. | 209 // One additional line start at the end, see [SourceFile.lineStarts]. |
| 201 lineStarts.add(stringOffset + 1); | 210 lineStarts.add(stringOffset + 1); |
| 202 file.lineStarts = lineStarts; | 211 file.lineStarts = lineStarts; |
| 203 } | 212 } |
| 204 | 213 |
| 205 return firstToken(); | 214 return firstToken(); |
| 206 } | 215 } |
| (...skipping 707 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 914 } | 923 } |
| 915 if (identical(next, $EOF)) return next; | 924 if (identical(next, $EOF)) return next; |
| 916 next = advance(); // Move past the $STX. | 925 next = advance(); // Move past the $STX. |
| 917 beginToken(); // The string interpolation suffix starts here. | 926 beginToken(); // The string interpolation suffix starts here. |
| 918 return next; | 927 return next; |
| 919 } | 928 } |
| 920 | 929 |
| 921 int tokenizeInterpolatedIdentifier(int next) { | 930 int tokenizeInterpolatedIdentifier(int next) { |
| 922 appendPrecedenceToken(STRING_INTERPOLATION_IDENTIFIER_INFO); | 931 appendPrecedenceToken(STRING_INTERPOLATION_IDENTIFIER_INFO); |
| 923 beginToken(); // The identifier starts here. | 932 beginToken(); // The identifier starts here. |
| 924 next = tokenizeKeywordOrIdentifier(next, false); | 933 |
| 934 if ($a <= next && next <= $z) { | |
| 935 next = tokenizeKeywordOrIdentifier(next, false); | |
| 936 } else if (($A <= next && next <= $Z) || identical(next, $_)) { | |
| 937 next = tokenizeIdentifier(next, scanOffset, false); | |
| 938 } else { | |
| 939 error("expected identifier or '{'", shouldAdvance: false); | |
| 940 } | |
| 925 beginToken(); // The string interpolation suffix starts here. | 941 beginToken(); // The string interpolation suffix starts here. |
| 926 return next; | 942 return next; |
| 927 } | 943 } |
| 928 | 944 |
| 929 int tokenizeSingleLineRawString(int next, int quoteChar, int start) { | 945 int tokenizeSingleLineRawString(int next, int quoteChar, int start) { |
| 930 bool asciiOnly = true; | 946 bool asciiOnly = true; |
| 931 next = advance(); // Advance past the quote. | 947 next = advance(); // Advance past the quote. |
| 932 while (next != $EOF) { | 948 while (next != $EOF) { |
| 933 if (identical(next, quoteChar)) { | 949 if (identical(next, quoteChar)) { |
| 934 if (!asciiOnly) handleUnicode(start); | 950 if (!asciiOnly) handleUnicode(start); |
| (...skipping 93 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 1028 } else if (next > 127) { | 1044 } else if (next > 127) { |
| 1029 asciiOnlyString = false; | 1045 asciiOnlyString = false; |
| 1030 asciiOnlyLine = false; | 1046 asciiOnlyLine = false; |
| 1031 } | 1047 } |
| 1032 next = advance(); | 1048 next = advance(); |
| 1033 } | 1049 } |
| 1034 if (!asciiOnlyLine) handleUnicode(unicodeStart); | 1050 if (!asciiOnlyLine) handleUnicode(unicodeStart); |
| 1035 return error("unterminated string literal"); | 1051 return error("unterminated string literal"); |
| 1036 } | 1052 } |
| 1037 | 1053 |
| 1038 int error(String message) { | 1054 int error(String message, {bool shouldAdvance: true}) { |
| 1039 appendStringToken(BAD_INPUT_INFO, message); | 1055 appendStringToken(BAD_INPUT_INFO, message); |
| 1040 return advance(); // Ensure progress. | 1056 if (atEndOfFile()) return $EOF; |
| 1057 if (shouldAdvance) { | |
| 1058 return advance(); // Ensure progress. | |
| 1059 } else { | |
| 1060 return -1; | |
| 1061 } | |
| 1041 } | 1062 } |
| 1042 | 1063 |
| 1043 void unmatchedBeginGroup(BeginGroupToken begin) { | 1064 void unmatchedBeginGroup(BeginGroupToken begin) { |
| 1044 String error = 'unmatched "${begin.stringValue}"'; | 1065 String error = 'unmatched "${begin.stringValue}"'; |
| 1045 Token close = | 1066 Token close = |
| 1046 new StringToken.fromString( | 1067 new StringToken.fromString( |
| 1047 BAD_INPUT_INFO, error, begin.charOffset, canonicalize: true); | 1068 BAD_INPUT_INFO, error, begin.charOffset, canonicalize: true); |
| 1048 | 1069 |
| 1049 // We want to ensure that unmatched BeginGroupTokens are reported | 1070 // We want to ensure that unmatched BeginGroupTokens are reported |
| 1050 // as errors. However, the rest of the parser assumes the groups | 1071 // as errors. However, the rest of the parser assumes the groups |
| 1051 // are well-balanced and will never look at the endGroup | 1072 // are well-balanced and will never look at the endGroup |
| 1052 // token. This is a nice property that allows us to skip quickly | 1073 // token. This is a nice property that allows us to skip quickly |
| 1053 // over correct code. By inserting an additional error token in | 1074 // over correct code. By inserting an additional error token in |
| 1054 // the stream, we can keep ignoring endGroup tokens. | 1075 // the stream, we can keep ignoring endGroup tokens. |
| 1055 // | 1076 // |
| 1056 // [begin] --next--> [tail] | 1077 // [begin] --next--> [tail] |
| 1057 // [begin] --endG--> [close] --next--> [next] --next--> [tail] | 1078 // [begin] --endG--> [close] --next--> [next] --next--> [tail] |
| 1058 // | 1079 // |
| 1059 // This allows the parser to skip from [begin] via endGroup to [close] and | 1080 // This allows the parser to skip from [begin] via endGroup to [close] and |
| 1060 // ignore the [close] token (assuming it's correct), then the error will be | 1081 // ignore the [close] token (assuming it's correct), then the error will be |
| 1061 // reported when parsing the [next] token. | 1082 // reported when parsing the [next] token. |
| 1062 | 1083 |
| 1063 Token next = new StringToken.fromString( | 1084 Token next = new StringToken.fromString( |
| 1064 BAD_INPUT_INFO, error, begin.charOffset, canonicalize: true); | 1085 BAD_INPUT_INFO, error, begin.charOffset, canonicalize: true); |
| 1065 begin.endGroup = close; | 1086 begin.endGroup = close; |
| 1066 close.next = next; | 1087 close.next = next; |
| 1067 next.next = begin.next; | 1088 next.next = begin.next; |
| 1068 } | 1089 } |
| 1069 } | 1090 } |
| OLD | NEW |