| OLD | NEW |
| 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 library fasta.scanner.abstract_scanner; | 5 library fasta.scanner.abstract_scanner; |
| 6 | 6 |
| 7 import 'dart:collection' show ListMixin; | 7 import 'dart:collection' show ListMixin; |
| 8 | 8 |
| 9 import 'dart:typed_data' show Uint16List, Uint32List; | 9 import 'dart:typed_data' show Uint16List, Uint32List; |
| 10 | 10 |
| (...skipping 168 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 179 /** Documentation in subclass [ArrayBasedScanner]. */ | 179 /** Documentation in subclass [ArrayBasedScanner]. */ |
| 180 int appendEndGroup(PrecedenceInfo info, int openKind); | 180 int appendEndGroup(PrecedenceInfo info, int openKind); |
| 181 | 181 |
| 182 /** Documentation in subclass [ArrayBasedScanner]. */ | 182 /** Documentation in subclass [ArrayBasedScanner]. */ |
| 183 void appendGt(PrecedenceInfo info); | 183 void appendGt(PrecedenceInfo info); |
| 184 | 184 |
| 185 /** Documentation in subclass [ArrayBasedScanner]. */ | 185 /** Documentation in subclass [ArrayBasedScanner]. */ |
| 186 void appendGtGt(PrecedenceInfo info); | 186 void appendGtGt(PrecedenceInfo info); |
| 187 | 187 |
| 188 /** Documentation in subclass [ArrayBasedScanner]. */ | 188 /** Documentation in subclass [ArrayBasedScanner]. */ |
| 189 void appendComment(start, bool asciiOnly); | 189 void appendComment(start, PrecedenceInfo info, bool asciiOnly); |
| 190 | 190 |
| 191 /// Append [token] to the token stream. | 191 /// Append [token] to the token stream. |
| 192 void appendErrorToken(ErrorToken token); | 192 void appendErrorToken(ErrorToken token); |
| 193 | 193 |
| 194 /** Documentation in subclass [ArrayBasedScanner]. */ | 194 /** Documentation in subclass [ArrayBasedScanner]. */ |
| 195 void discardOpenLt(); | 195 void discardOpenLt(); |
| 196 | 196 |
| 197 /// Return true when at EOF. | 197 /// Return true when at EOF. |
| 198 bool atEndOfFile(); | 198 bool atEndOfFile(); |
| 199 | 199 |
| (...skipping 72 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 272 } | 272 } |
| 273 | 273 |
| 274 if (identical(next, $CLOSE_CURLY_BRACKET)) { | 274 if (identical(next, $CLOSE_CURLY_BRACKET)) { |
| 275 return appendEndGroup(CLOSE_CURLY_BRACKET_INFO, OPEN_CURLY_BRACKET_TOKEN); | 275 return appendEndGroup(CLOSE_CURLY_BRACKET_INFO, OPEN_CURLY_BRACKET_TOKEN); |
| 276 } | 276 } |
| 277 | 277 |
| 278 if (identical(next, $SLASH)) { | 278 if (identical(next, $SLASH)) { |
| 279 return tokenizeSlashOrComment(next); | 279 return tokenizeSlashOrComment(next); |
| 280 } | 280 } |
| 281 | 281 |
| 282 | |
| 283 if (identical(next, $OPEN_CURLY_BRACKET)) { | 282 if (identical(next, $OPEN_CURLY_BRACKET)) { |
| 284 appendBeginGroup(OPEN_CURLY_BRACKET_INFO); | 283 appendBeginGroup(OPEN_CURLY_BRACKET_INFO); |
| 285 return advance(); | 284 return advance(); |
| 286 } | 285 } |
| 287 | 286 |
| 288 if (identical(next, $DQ) || identical(next, $SQ)) { | 287 if (identical(next, $DQ) || identical(next, $SQ)) { |
| 289 return tokenizeString(next, scanOffset, false); | 288 return tokenizeString(next, scanOffset, false); |
| 290 } | 289 } |
| 291 | 290 |
| 292 if(identical(next, $_)){ | 291 if (identical(next, $_)) { |
| 293 return tokenizeKeywordOrIdentifier(next, true); | 292 return tokenizeKeywordOrIdentifier(next, true); |
| 294 } | 293 } |
| 295 | 294 |
| 296 if (identical(next, $COLON)) { | 295 if (identical(next, $COLON)) { |
| 297 appendPrecedenceToken(COLON_INFO); | 296 appendPrecedenceToken(COLON_INFO); |
| 298 return advance(); | 297 return advance(); |
| 299 } | 298 } |
| 300 | 299 |
| 301 if (identical(next, $LT)) { | 300 if (identical(next, $LT)) { |
| 302 return tokenizeLessThan(next); | 301 return tokenizeLessThan(next); |
| (...skipping 37 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 340 } | 339 } |
| 341 | 340 |
| 342 if (identical(next, $BAR)) { | 341 if (identical(next, $BAR)) { |
| 343 return tokenizeBar(next); | 342 return tokenizeBar(next); |
| 344 } | 343 } |
| 345 | 344 |
| 346 if (identical(next, $PLUS)) { | 345 if (identical(next, $PLUS)) { |
| 347 return tokenizePlus(next); | 346 return tokenizePlus(next); |
| 348 } | 347 } |
| 349 | 348 |
| 350 if(identical(next, $$)){ | 349 if (identical(next, $$)) { |
| 351 return tokenizeKeywordOrIdentifier(next, true); | 350 return tokenizeKeywordOrIdentifier(next, true); |
| 352 } | 351 } |
| 353 | 352 |
| 354 if (identical(next, $MINUS)) { | 353 if (identical(next, $MINUS)) { |
| 355 return tokenizeMinus(next); | 354 return tokenizeMinus(next); |
| 356 } | 355 } |
| 357 | 356 |
| 358 if (identical(next, $STAR)) { | 357 if (identical(next, $STAR)) { |
| 359 return tokenizeMultiply(next); | 358 return tokenizeMultiply(next); |
| 360 } | 359 } |
| (...skipping 362 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 723 | 722 |
| 724 int tokenizeSingleLineComment(int next, int start) { | 723 int tokenizeSingleLineComment(int next, int start) { |
| 725 bool asciiOnly = true; | 724 bool asciiOnly = true; |
| 726 while (true) { | 725 while (true) { |
| 727 next = advance(); | 726 next = advance(); |
| 728 if (next > 127) asciiOnly = false; | 727 if (next > 127) asciiOnly = false; |
| 729 if (identical($LF, next) || | 728 if (identical($LF, next) || |
| 730 identical($CR, next) || | 729 identical($CR, next) || |
| 731 identical($EOF, next)) { | 730 identical($EOF, next)) { |
| 732 if (!asciiOnly) handleUnicode(start); | 731 if (!asciiOnly) handleUnicode(start); |
| 733 appendComment(start, asciiOnly); | 732 appendComment(start, SINGLE_LINE_COMMENT_INFO, asciiOnly); |
| 734 return next; | 733 return next; |
| 735 } | 734 } |
| 736 } | 735 } |
| 737 } | 736 } |
| 738 | 737 |
| 739 int tokenizeMultiLineComment(int next, int start) { | 738 int tokenizeMultiLineComment(int next, int start) { |
| 740 bool asciiOnlyComment = true; // Track if the entire comment is ASCII. | 739 bool asciiOnlyComment = true; // Track if the entire comment is ASCII. |
| 741 bool asciiOnlyLines = true; // Track ASCII since the last handleUnicode. | 740 bool asciiOnlyLines = true; // Track ASCII since the last handleUnicode. |
| 742 int unicodeStart = start; | 741 int unicodeStart = start; |
| 743 int nesting = 1; | 742 int nesting = 1; |
| 744 next = advance(); | 743 next = advance(); |
| 745 while (true) { | 744 while (true) { |
| 746 if (identical($EOF, next)) { | 745 if (identical($EOF, next)) { |
| 747 if (!asciiOnlyLines) handleUnicode(unicodeStart); | 746 if (!asciiOnlyLines) handleUnicode(unicodeStart); |
| 748 unterminated('/*'); | 747 unterminated('/*'); |
| 749 break; | 748 break; |
| 750 } else if (identical($STAR, next)) { | 749 } else if (identical($STAR, next)) { |
| 751 next = advance(); | 750 next = advance(); |
| 752 if (identical($SLASH, next)) { | 751 if (identical($SLASH, next)) { |
| 753 --nesting; | 752 --nesting; |
| 754 if (0 == nesting) { | 753 if (0 == nesting) { |
| 755 if (!asciiOnlyLines) handleUnicode(unicodeStart); | 754 if (!asciiOnlyLines) handleUnicode(unicodeStart); |
| 756 next = advance(); | 755 next = advance(); |
| 757 appendComment(start, asciiOnlyComment); | 756 appendComment(start, MULTI_LINE_COMMENT_INFO, asciiOnlyComment); |
| 758 break; | 757 break; |
| 759 } else { | 758 } else { |
| 760 next = advance(); | 759 next = advance(); |
| 761 } | 760 } |
| 762 } | 761 } |
| 763 } else if (identical($SLASH, next)) { | 762 } else if (identical($SLASH, next)) { |
| 764 next = advance(); | 763 next = advance(); |
| 765 if (identical($STAR, next)) { | 764 if (identical($STAR, next)) { |
| 766 next = advance(); | 765 next = advance(); |
| 767 ++nesting; | 766 ++nesting; |
| (...skipping 468 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1236 switchToUint32(newLength); | 1235 switchToUint32(newLength); |
| 1237 } | 1236 } |
| 1238 } | 1237 } |
| 1239 | 1238 |
| 1240 void switchToUint32(int newLength) { | 1239 void switchToUint32(int newLength) { |
| 1241 final newArray = new Uint32List(newLength); | 1240 final newArray = new Uint32List(newLength); |
| 1242 newArray.setRange(0, arrayLength, array); | 1241 newArray.setRange(0, arrayLength, array); |
| 1243 array = newArray; | 1242 array = newArray; |
| 1244 } | 1243 } |
| 1245 } | 1244 } |
| OLD | NEW |