Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(157)

Side by Side Diff: pkg/front_end/lib/src/fasta/scanner/abstract_scanner.dart

Issue 2837473002: merge fasta.PrecedenceInfo into analyzer.TokenType (Closed)
Patch Set: rebase Created 3 years, 8 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file
2 // for details. All rights reserved. Use of this source code is governed by a 2 // for details. All rights reserved. Use of this source code is governed by a
3 // BSD-style license that can be found in the LICENSE file. 3 // BSD-style license that can be found in the LICENSE file.
4 4
5 library fasta.scanner.abstract_scanner; 5 library fasta.scanner.abstract_scanner;
6 6
7 import 'dart:collection' show ListMixin; 7 import 'dart:collection' show ListMixin;
8 8
9 import 'dart:typed_data' show Uint16List, Uint32List; 9 import 'dart:typed_data' show Uint16List, Uint32List;
10 10
11 import '../../scanner/token.dart' show TokenType;
12
11 import '../scanner.dart' 13 import '../scanner.dart'
12 show ErrorToken, Keyword, Scanner, buildUnexpectedCharacterToken; 14 show ErrorToken, Keyword, Scanner, buildUnexpectedCharacterToken;
13 15
14 import 'error_token.dart' show UnterminatedToken; 16 import 'error_token.dart' show UnterminatedToken;
15 17
16 import 'keyword_state.dart' show KeywordState; 18 import 'keyword_state.dart' show KeywordState;
17 19
18 import 'precedence.dart'; 20 import 'precedence.dart';
19 21
20 import 'token.dart' 22 import 'token.dart'
(...skipping 131 matching lines...) Expand 10 before | Expand all | Expand 10 after
152 154
153 /** 155 /**
154 * Appends a substring from the scan offset [:start:] to the current 156 * Appends a substring from the scan offset [:start:] to the current
155 * [:scanOffset:] plus the [:extraOffset:]. For example, if the current 157 * [:scanOffset:] plus the [:extraOffset:]. For example, if the current
156 * scanOffset is 10, then [:appendSubstringToken(5, -1):] will append the 158 * scanOffset is 10, then [:appendSubstringToken(5, -1):] will append the
157 * substring string [5,9). 159 * substring string [5,9).
158 * 160 *
159 * Note that [extraOffset] can only be used if the covered character(s) are 161 * Note that [extraOffset] can only be used if the covered character(s) are
160 * known to be ASCII. 162 * known to be ASCII.
161 */ 163 */
162 void appendSubstringToken(PrecedenceInfo info, int start, bool asciiOnly, 164 void appendSubstringToken(TokenType info, int start, bool asciiOnly,
163 [int extraOffset]); 165 [int extraOffset]);
164 166
165 /** Documentation in subclass [ArrayBasedScanner]. */ 167 /** Documentation in subclass [ArrayBasedScanner]. */
166 void appendPrecedenceToken(PrecedenceInfo info); 168 void appendPrecedenceToken(TokenType info);
167 169
168 /** Documentation in subclass [ArrayBasedScanner]. */ 170 /** Documentation in subclass [ArrayBasedScanner]. */
169 int select(int choice, PrecedenceInfo yes, PrecedenceInfo no); 171 int select(int choice, TokenType yes, TokenType no);
170 172
171 /** Documentation in subclass [ArrayBasedScanner]. */ 173 /** Documentation in subclass [ArrayBasedScanner]. */
172 void appendKeywordToken(Keyword keyword); 174 void appendKeywordToken(Keyword keyword);
173 175
174 /** Documentation in subclass [ArrayBasedScanner]. */ 176 /** Documentation in subclass [ArrayBasedScanner]. */
175 void appendEofToken(); 177 void appendEofToken();
176 178
177 /** Documentation in subclass [ArrayBasedScanner]. */ 179 /** Documentation in subclass [ArrayBasedScanner]. */
178 void appendWhiteSpace(int next); 180 void appendWhiteSpace(int next);
179 181
180 /** Documentation in subclass [ArrayBasedScanner]. */ 182 /** Documentation in subclass [ArrayBasedScanner]. */
181 void lineFeedInMultiline(); 183 void lineFeedInMultiline();
182 184
183 /** Documentation in subclass [ArrayBasedScanner]. */ 185 /** Documentation in subclass [ArrayBasedScanner]. */
184 void appendBeginGroup(PrecedenceInfo info); 186 void appendBeginGroup(TokenType info);
185 187
186 /** Documentation in subclass [ArrayBasedScanner]. */ 188 /** Documentation in subclass [ArrayBasedScanner]. */
187 int appendEndGroup(PrecedenceInfo info, int openKind); 189 int appendEndGroup(TokenType info, int openKind);
188 190
189 /** Documentation in subclass [ArrayBasedScanner]. */ 191 /** Documentation in subclass [ArrayBasedScanner]. */
190 void appendGt(PrecedenceInfo info); 192 void appendGt(TokenType info);
191 193
192 /** Documentation in subclass [ArrayBasedScanner]. */ 194 /** Documentation in subclass [ArrayBasedScanner]. */
193 void appendGtGt(PrecedenceInfo info); 195 void appendGtGt(TokenType info);
194 196
195 /// Append [token] to the token stream. 197 /// Append [token] to the token stream.
196 void appendErrorToken(ErrorToken token); 198 void appendErrorToken(ErrorToken token);
197 199
198 /** 200 /**
199 * Returns a new comment from the scan offset [start] to the current 201 * Returns a new comment from the scan offset [start] to the current
200 * [scanOffset] plus the [extraOffset]. For example, if the current 202 * [scanOffset] plus the [extraOffset]. For example, if the current
201 * scanOffset is 10, then [appendSubstringToken(5, -1)] will append the 203 * scanOffset is 10, then [appendSubstringToken(5, -1)] will append the
202 * substring string [5,9). 204 * substring string [5,9).
203 * 205 *
204 * Note that [extraOffset] can only be used if the covered character(s) are 206 * Note that [extraOffset] can only be used if the covered character(s) are
205 * known to be ASCII. 207 * known to be ASCII.
206 */ 208 */
207 CommentToken createCommentToken( 209 CommentToken createCommentToken(TokenType info, int start, bool asciiOnly,
208 PrecedenceInfo info, int start, bool asciiOnly,
209 [int extraOffset = 0]); 210 [int extraOffset = 0]);
210 211
211 /** 212 /**
212 * Returns a new dartdoc from the scan offset [start] to the current 213 * Returns a new dartdoc from the scan offset [start] to the current
213 * [scanOffset] plus the [extraOffset]. For example, if the current 214 * [scanOffset] plus the [extraOffset]. For example, if the current
214 * scanOffset is 10, then [appendSubstringToken(5, -1)] will append the 215 * scanOffset is 10, then [appendSubstringToken(5, -1)] will append the
215 * substring string [5,9). 216 * substring string [5,9).
216 * 217 *
217 * Note that [extraOffset] can only be used if the covered character(s) are 218 * Note that [extraOffset] can only be used if the covered character(s) are
218 * known to be ASCII. 219 * known to be ASCII.
219 */ 220 */
220 DartDocToken createDartDocToken( 221 DartDocToken createDartDocToken(TokenType info, int start, bool asciiOnly,
221 PrecedenceInfo info, int start, bool asciiOnly,
222 [int extraOffset = 0]); 222 [int extraOffset = 0]);
223 223
224 /** Documentation in subclass [ArrayBasedScanner]. */ 224 /** Documentation in subclass [ArrayBasedScanner]. */
225 void discardOpenLt(); 225 void discardOpenLt();
226 226
227 /// Return true when at EOF. 227 /// Return true when at EOF.
228 bool atEndOfFile(); 228 bool atEndOfFile();
229 229
230 Token tokenize() { 230 Token tokenize() {
231 while (!atEndOfFile()) { 231 while (!atEndOfFile()) {
(...skipping 586 matching lines...) Expand 10 before | Expand all | Expand 10 after
818 if (next > 127) { 818 if (next > 127) {
819 asciiOnlyLines = false; 819 asciiOnlyLines = false;
820 asciiOnlyComment = false; 820 asciiOnlyComment = false;
821 } 821 }
822 next = advance(); 822 next = advance();
823 } 823 }
824 } 824 }
825 return next; 825 return next;
826 } 826 }
827 827
828 void appendComment(int start, PrecedenceInfo info, bool asciiOnly) { 828 void appendComment(int start, TokenType info, bool asciiOnly) {
829 if (!includeComments) return; 829 if (!includeComments) return;
830 CommentToken newComment = createCommentToken(info, start, asciiOnly); 830 CommentToken newComment = createCommentToken(info, start, asciiOnly);
831 if (scanGenericMethodComments) { 831 if (scanGenericMethodComments) {
832 String value = newComment.lexeme; 832 String value = newComment.lexeme;
833 int length = value.length; 833 int length = value.length;
834 if (length > 5 && 834 if (length > 5 &&
835 value.codeUnitAt(0) == $SLASH && 835 value.codeUnitAt(0) == $SLASH &&
836 value.codeUnitAt(1) == $STAR && 836 value.codeUnitAt(1) == $STAR &&
837 value.codeUnitAt(2) == $EQ) { 837 value.codeUnitAt(2) == $EQ) {
838 newComment = new CommentToken.fromString( 838 newComment = new CommentToken.fromString(
839 GENERIC_METHOD_TYPE_ASSIGN, value, start); 839 GENERIC_METHOD_TYPE_ASSIGN, value, start);
840 } else if (length > 6 && 840 } else if (length > 6 &&
841 value.codeUnitAt(0) == $SLASH && 841 value.codeUnitAt(0) == $SLASH &&
842 value.codeUnitAt(1) == $STAR && 842 value.codeUnitAt(1) == $STAR &&
843 value.codeUnitAt(2) == $LT && 843 value.codeUnitAt(2) == $LT &&
844 value.codeUnitAt(length - 1) == $SLASH && 844 value.codeUnitAt(length - 1) == $SLASH &&
845 value.codeUnitAt(length - 2) == $STAR && 845 value.codeUnitAt(length - 2) == $STAR &&
846 value.codeUnitAt(length - 3) == $GT) { 846 value.codeUnitAt(length - 3) == $GT) {
847 newComment = 847 newComment =
848 new CommentToken.fromString(GENERIC_METHOD_TYPE_LIST, value, start); 848 new CommentToken.fromString(GENERIC_METHOD_TYPE_LIST, value, start);
849 } 849 }
850 } 850 }
851 _appendToCommentStream(newComment); 851 _appendToCommentStream(newComment);
852 } 852 }
853 853
854 void appendDartDoc(int start, PrecedenceInfo info, bool asciiOnly) { 854 void appendDartDoc(int start, TokenType info, bool asciiOnly) {
855 if (!includeComments) return; 855 if (!includeComments) return;
856 Token newComment = createDartDocToken(info, start, asciiOnly); 856 Token newComment = createDartDocToken(info, start, asciiOnly);
857 _appendToCommentStream(newComment); 857 _appendToCommentStream(newComment);
858 } 858 }
859 859
860 /** 860 /**
861 * Append the given token to the [tail] of the current stream of tokens. 861 * Append the given token to the [tail] of the current stream of tokens.
862 */ 862 */
863 void appendToken(Token token) { 863 void appendToken(Token token) {
864 tail.next = token; 864 tail.next = token;
(...skipping 333 matching lines...) Expand 10 before | Expand all | Expand 10 after
1198 int advanceAfterError(bool shouldAdvance) { 1198 int advanceAfterError(bool shouldAdvance) {
1199 if (atEndOfFile()) return $EOF; 1199 if (atEndOfFile()) return $EOF;
1200 if (shouldAdvance) { 1200 if (shouldAdvance) {
1201 return advance(); // Ensure progress. 1201 return advance(); // Ensure progress.
1202 } else { 1202 } else {
1203 return -1; 1203 return -1;
1204 } 1204 }
1205 } 1205 }
1206 } 1206 }
1207 1207
1208 PrecedenceInfo closeBraceInfoFor(BeginGroupToken begin) { 1208 TokenType closeBraceInfoFor(BeginGroupToken begin) {
1209 return const { 1209 return const {
1210 '(': CLOSE_PAREN_INFO, 1210 '(': CLOSE_PAREN_INFO,
1211 '[': CLOSE_SQUARE_BRACKET_INFO, 1211 '[': CLOSE_SQUARE_BRACKET_INFO,
1212 '{': CLOSE_CURLY_BRACKET_INFO, 1212 '{': CLOSE_CURLY_BRACKET_INFO,
1213 '<': GT_INFO, 1213 '<': GT_INFO,
1214 r'${': CLOSE_CURLY_BRACKET_INFO, 1214 r'${': CLOSE_CURLY_BRACKET_INFO,
1215 }[begin.lexeme]; 1215 }[begin.lexeme];
1216 } 1216 }
1217 1217
1218 class LineStarts extends Object with ListMixin<int> { 1218 class LineStarts extends Object with ListMixin<int> {
(...skipping 65 matching lines...) Expand 10 before | Expand all | Expand 10 after
1284 switchToUint32(newLength); 1284 switchToUint32(newLength);
1285 } 1285 }
1286 } 1286 }
1287 1287
1288 void switchToUint32(int newLength) { 1288 void switchToUint32(int newLength) {
1289 final newArray = new Uint32List(newLength); 1289 final newArray = new Uint32List(newLength);
1290 newArray.setRange(0, arrayLength, array); 1290 newArray.setRange(0, arrayLength, array);
1291 array = newArray; 1291 array = newArray;
1292 } 1292 }
1293 } 1293 }
OLDNEW
« no previous file with comments | « pkg/front_end/lib/src/fasta/parser/parser.dart ('k') | pkg/front_end/lib/src/fasta/scanner/array_based_scanner.dart » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698