Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(588)

Side by Side Diff: pkg/front_end/lib/src/fasta/scanner/abstract_scanner.dart

Issue 2757153002: update fasta scanner to generate two types of comments (Closed)
Patch Set: rebase Created 3 years, 9 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file
2 // for details. All rights reserved. Use of this source code is governed by a 2 // for details. All rights reserved. Use of this source code is governed by a
3 // BSD-style license that can be found in the LICENSE file. 3 // BSD-style license that can be found in the LICENSE file.
4 4
5 library fasta.scanner.abstract_scanner; 5 library fasta.scanner.abstract_scanner;
6 6
7 import 'dart:collection' show ListMixin; 7 import 'dart:collection' show ListMixin;
8 8
9 import 'dart:typed_data' show Uint16List, Uint32List; 9 import 'dart:typed_data' show Uint16List, Uint32List;
10 10
(...skipping 168 matching lines...) Expand 10 before | Expand all | Expand 10 after
179 /** Documentation in subclass [ArrayBasedScanner]. */ 179 /** Documentation in subclass [ArrayBasedScanner]. */
180 int appendEndGroup(PrecedenceInfo info, int openKind); 180 int appendEndGroup(PrecedenceInfo info, int openKind);
181 181
182 /** Documentation in subclass [ArrayBasedScanner]. */ 182 /** Documentation in subclass [ArrayBasedScanner]. */
183 void appendGt(PrecedenceInfo info); 183 void appendGt(PrecedenceInfo info);
184 184
185 /** Documentation in subclass [ArrayBasedScanner]. */ 185 /** Documentation in subclass [ArrayBasedScanner]. */
186 void appendGtGt(PrecedenceInfo info); 186 void appendGtGt(PrecedenceInfo info);
187 187
188 /** Documentation in subclass [ArrayBasedScanner]. */ 188 /** Documentation in subclass [ArrayBasedScanner]. */
189 void appendComment(start, bool asciiOnly); 189 void appendComment(start, PrecedenceInfo info, bool asciiOnly);
190 190
191 /// Append [token] to the token stream. 191 /// Append [token] to the token stream.
192 void appendErrorToken(ErrorToken token); 192 void appendErrorToken(ErrorToken token);
193 193
194 /** Documentation in subclass [ArrayBasedScanner]. */ 194 /** Documentation in subclass [ArrayBasedScanner]. */
195 void discardOpenLt(); 195 void discardOpenLt();
196 196
197 /// Return true when at EOF. 197 /// Return true when at EOF.
198 bool atEndOfFile(); 198 bool atEndOfFile();
199 199
(...skipping 72 matching lines...) Expand 10 before | Expand all | Expand 10 after
272 } 272 }
273 273
274 if (identical(next, $CLOSE_CURLY_BRACKET)) { 274 if (identical(next, $CLOSE_CURLY_BRACKET)) {
275 return appendEndGroup(CLOSE_CURLY_BRACKET_INFO, OPEN_CURLY_BRACKET_TOKEN); 275 return appendEndGroup(CLOSE_CURLY_BRACKET_INFO, OPEN_CURLY_BRACKET_TOKEN);
276 } 276 }
277 277
278 if (identical(next, $SLASH)) { 278 if (identical(next, $SLASH)) {
279 return tokenizeSlashOrComment(next); 279 return tokenizeSlashOrComment(next);
280 } 280 }
281 281
282
283 if (identical(next, $OPEN_CURLY_BRACKET)) { 282 if (identical(next, $OPEN_CURLY_BRACKET)) {
284 appendBeginGroup(OPEN_CURLY_BRACKET_INFO); 283 appendBeginGroup(OPEN_CURLY_BRACKET_INFO);
285 return advance(); 284 return advance();
286 } 285 }
287 286
288 if (identical(next, $DQ) || identical(next, $SQ)) { 287 if (identical(next, $DQ) || identical(next, $SQ)) {
289 return tokenizeString(next, scanOffset, false); 288 return tokenizeString(next, scanOffset, false);
290 } 289 }
291 290
292 if(identical(next, $_)){ 291 if (identical(next, $_)) {
293 return tokenizeKeywordOrIdentifier(next, true); 292 return tokenizeKeywordOrIdentifier(next, true);
294 } 293 }
295 294
296 if (identical(next, $COLON)) { 295 if (identical(next, $COLON)) {
297 appendPrecedenceToken(COLON_INFO); 296 appendPrecedenceToken(COLON_INFO);
298 return advance(); 297 return advance();
299 } 298 }
300 299
301 if (identical(next, $LT)) { 300 if (identical(next, $LT)) {
302 return tokenizeLessThan(next); 301 return tokenizeLessThan(next);
(...skipping 37 matching lines...) Expand 10 before | Expand all | Expand 10 after
340 } 339 }
341 340
342 if (identical(next, $BAR)) { 341 if (identical(next, $BAR)) {
343 return tokenizeBar(next); 342 return tokenizeBar(next);
344 } 343 }
345 344
346 if (identical(next, $PLUS)) { 345 if (identical(next, $PLUS)) {
347 return tokenizePlus(next); 346 return tokenizePlus(next);
348 } 347 }
349 348
350 if(identical(next, $$)){ 349 if (identical(next, $$)) {
351 return tokenizeKeywordOrIdentifier(next, true); 350 return tokenizeKeywordOrIdentifier(next, true);
352 } 351 }
353 352
354 if (identical(next, $MINUS)) { 353 if (identical(next, $MINUS)) {
355 return tokenizeMinus(next); 354 return tokenizeMinus(next);
356 } 355 }
357 356
358 if (identical(next, $STAR)) { 357 if (identical(next, $STAR)) {
359 return tokenizeMultiply(next); 358 return tokenizeMultiply(next);
360 } 359 }
(...skipping 362 matching lines...) Expand 10 before | Expand all | Expand 10 after
723 722
724 int tokenizeSingleLineComment(int next, int start) { 723 int tokenizeSingleLineComment(int next, int start) {
725 bool asciiOnly = true; 724 bool asciiOnly = true;
726 while (true) { 725 while (true) {
727 next = advance(); 726 next = advance();
728 if (next > 127) asciiOnly = false; 727 if (next > 127) asciiOnly = false;
729 if (identical($LF, next) || 728 if (identical($LF, next) ||
730 identical($CR, next) || 729 identical($CR, next) ||
731 identical($EOF, next)) { 730 identical($EOF, next)) {
732 if (!asciiOnly) handleUnicode(start); 731 if (!asciiOnly) handleUnicode(start);
733 appendComment(start, asciiOnly); 732 appendComment(start, SINGLE_LINE_COMMENT_INFO, asciiOnly);
734 return next; 733 return next;
735 } 734 }
736 } 735 }
737 } 736 }
738 737
739 int tokenizeMultiLineComment(int next, int start) { 738 int tokenizeMultiLineComment(int next, int start) {
740 bool asciiOnlyComment = true; // Track if the entire comment is ASCII. 739 bool asciiOnlyComment = true; // Track if the entire comment is ASCII.
741 bool asciiOnlyLines = true; // Track ASCII since the last handleUnicode. 740 bool asciiOnlyLines = true; // Track ASCII since the last handleUnicode.
742 int unicodeStart = start; 741 int unicodeStart = start;
743 int nesting = 1; 742 int nesting = 1;
744 next = advance(); 743 next = advance();
745 while (true) { 744 while (true) {
746 if (identical($EOF, next)) { 745 if (identical($EOF, next)) {
747 if (!asciiOnlyLines) handleUnicode(unicodeStart); 746 if (!asciiOnlyLines) handleUnicode(unicodeStart);
748 unterminated('/*'); 747 unterminated('/*');
749 break; 748 break;
750 } else if (identical($STAR, next)) { 749 } else if (identical($STAR, next)) {
751 next = advance(); 750 next = advance();
752 if (identical($SLASH, next)) { 751 if (identical($SLASH, next)) {
753 --nesting; 752 --nesting;
754 if (0 == nesting) { 753 if (0 == nesting) {
755 if (!asciiOnlyLines) handleUnicode(unicodeStart); 754 if (!asciiOnlyLines) handleUnicode(unicodeStart);
756 next = advance(); 755 next = advance();
757 appendComment(start, asciiOnlyComment); 756 appendComment(start, MULTI_LINE_COMMENT_INFO, asciiOnlyComment);
758 break; 757 break;
759 } else { 758 } else {
760 next = advance(); 759 next = advance();
761 } 760 }
762 } 761 }
763 } else if (identical($SLASH, next)) { 762 } else if (identical($SLASH, next)) {
764 next = advance(); 763 next = advance();
765 if (identical($STAR, next)) { 764 if (identical($STAR, next)) {
766 next = advance(); 765 next = advance();
767 ++nesting; 766 ++nesting;
(...skipping 468 matching lines...) Expand 10 before | Expand all | Expand 10 after
1236 switchToUint32(newLength); 1235 switchToUint32(newLength);
1237 } 1236 }
1238 } 1237 }
1239 1238
1240 void switchToUint32(int newLength) { 1239 void switchToUint32(int newLength) {
1241 final newArray = new Uint32List(newLength); 1240 final newArray = new Uint32List(newLength);
1242 newArray.setRange(0, arrayLength, array); 1241 newArray.setRange(0, arrayLength, array);
1243 array = newArray; 1242 array = newArray;
1244 } 1243 }
1245 } 1244 }
OLDNEW
« no previous file with comments | « pkg/analyzer/lib/src/fasta/token_utils.dart ('k') | pkg/front_end/lib/src/fasta/scanner/array_based_scanner.dart » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698