| OLD | NEW |
| 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 library fasta.scanner.abstract_scanner; | 5 library fasta.scanner.abstract_scanner; |
| 6 | 6 |
| 7 import 'dart:collection' show ListMixin; | 7 import 'dart:collection' show ListMixin; |
| 8 | 8 |
| 9 import 'dart:typed_data' show Uint16List, Uint32List; | 9 import 'dart:typed_data' show Uint16List, Uint32List; |
| 10 | 10 |
| 11 import '../../scanner/token.dart' show TokenType; |
| 12 |
| 11 import '../scanner.dart' | 13 import '../scanner.dart' |
| 12 show ErrorToken, Keyword, Scanner, buildUnexpectedCharacterToken; | 14 show ErrorToken, Keyword, Scanner, buildUnexpectedCharacterToken; |
| 13 | 15 |
| 14 import 'error_token.dart' show UnterminatedToken; | 16 import 'error_token.dart' show UnterminatedToken; |
| 15 | 17 |
| 16 import 'keyword_state.dart' show KeywordState; | 18 import 'keyword_state.dart' show KeywordState; |
| 17 | 19 |
| 18 import 'precedence.dart'; | 20 import 'precedence.dart'; |
| 19 | 21 |
| 20 import 'token.dart' | 22 import 'token.dart' |
| (...skipping 131 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 152 | 154 |
| 153 /** | 155 /** |
| 154 * Appends a substring from the scan offset [:start:] to the current | 156 * Appends a substring from the scan offset [:start:] to the current |
| 155 * [:scanOffset:] plus the [:extraOffset:]. For example, if the current | 157 * [:scanOffset:] plus the [:extraOffset:]. For example, if the current |
| 156 * scanOffset is 10, then [:appendSubstringToken(5, -1):] will append the | 158 * scanOffset is 10, then [:appendSubstringToken(5, -1):] will append the |
| 157 * substring string [5,9). | 159 * substring string [5,9). |
| 158 * | 160 * |
| 159 * Note that [extraOffset] can only be used if the covered character(s) are | 161 * Note that [extraOffset] can only be used if the covered character(s) are |
| 160 * known to be ASCII. | 162 * known to be ASCII. |
| 161 */ | 163 */ |
| 162 void appendSubstringToken(PrecedenceInfo info, int start, bool asciiOnly, | 164 void appendSubstringToken(TokenType info, int start, bool asciiOnly, |
| 163 [int extraOffset]); | 165 [int extraOffset]); |
| 164 | 166 |
| 165 /** Documentation in subclass [ArrayBasedScanner]. */ | 167 /** Documentation in subclass [ArrayBasedScanner]. */ |
| 166 void appendPrecedenceToken(PrecedenceInfo info); | 168 void appendPrecedenceToken(TokenType info); |
| 167 | 169 |
| 168 /** Documentation in subclass [ArrayBasedScanner]. */ | 170 /** Documentation in subclass [ArrayBasedScanner]. */ |
| 169 int select(int choice, PrecedenceInfo yes, PrecedenceInfo no); | 171 int select(int choice, TokenType yes, TokenType no); |
| 170 | 172 |
| 171 /** Documentation in subclass [ArrayBasedScanner]. */ | 173 /** Documentation in subclass [ArrayBasedScanner]. */ |
| 172 void appendKeywordToken(Keyword keyword); | 174 void appendKeywordToken(Keyword keyword); |
| 173 | 175 |
| 174 /** Documentation in subclass [ArrayBasedScanner]. */ | 176 /** Documentation in subclass [ArrayBasedScanner]. */ |
| 175 void appendEofToken(); | 177 void appendEofToken(); |
| 176 | 178 |
| 177 /** Documentation in subclass [ArrayBasedScanner]. */ | 179 /** Documentation in subclass [ArrayBasedScanner]. */ |
| 178 void appendWhiteSpace(int next); | 180 void appendWhiteSpace(int next); |
| 179 | 181 |
| 180 /** Documentation in subclass [ArrayBasedScanner]. */ | 182 /** Documentation in subclass [ArrayBasedScanner]. */ |
| 181 void lineFeedInMultiline(); | 183 void lineFeedInMultiline(); |
| 182 | 184 |
| 183 /** Documentation in subclass [ArrayBasedScanner]. */ | 185 /** Documentation in subclass [ArrayBasedScanner]. */ |
| 184 void appendBeginGroup(PrecedenceInfo info); | 186 void appendBeginGroup(TokenType info); |
| 185 | 187 |
| 186 /** Documentation in subclass [ArrayBasedScanner]. */ | 188 /** Documentation in subclass [ArrayBasedScanner]. */ |
| 187 int appendEndGroup(PrecedenceInfo info, int openKind); | 189 int appendEndGroup(TokenType info, int openKind); |
| 188 | 190 |
| 189 /** Documentation in subclass [ArrayBasedScanner]. */ | 191 /** Documentation in subclass [ArrayBasedScanner]. */ |
| 190 void appendGt(PrecedenceInfo info); | 192 void appendGt(TokenType info); |
| 191 | 193 |
| 192 /** Documentation in subclass [ArrayBasedScanner]. */ | 194 /** Documentation in subclass [ArrayBasedScanner]. */ |
| 193 void appendGtGt(PrecedenceInfo info); | 195 void appendGtGt(TokenType info); |
| 194 | 196 |
| 195 /// Append [token] to the token stream. | 197 /// Append [token] to the token stream. |
| 196 void appendErrorToken(ErrorToken token); | 198 void appendErrorToken(ErrorToken token); |
| 197 | 199 |
| 198 /** | 200 /** |
| 199 * Returns a new comment from the scan offset [start] to the current | 201 * Returns a new comment from the scan offset [start] to the current |
| 200 * [scanOffset] plus the [extraOffset]. For example, if the current | 202 * [scanOffset] plus the [extraOffset]. For example, if the current |
| 201 * scanOffset is 10, then [appendSubstringToken(5, -1)] will append the | 203 * scanOffset is 10, then [appendSubstringToken(5, -1)] will append the |
| 202 * substring string [5,9). | 204 * substring string [5,9). |
| 203 * | 205 * |
| 204 * Note that [extraOffset] can only be used if the covered character(s) are | 206 * Note that [extraOffset] can only be used if the covered character(s) are |
| 205 * known to be ASCII. | 207 * known to be ASCII. |
| 206 */ | 208 */ |
| 207 CommentToken createCommentToken( | 209 CommentToken createCommentToken(TokenType info, int start, bool asciiOnly, |
| 208 PrecedenceInfo info, int start, bool asciiOnly, | |
| 209 [int extraOffset = 0]); | 210 [int extraOffset = 0]); |
| 210 | 211 |
| 211 /** | 212 /** |
| 212 * Returns a new dartdoc from the scan offset [start] to the current | 213 * Returns a new dartdoc from the scan offset [start] to the current |
| 213 * [scanOffset] plus the [extraOffset]. For example, if the current | 214 * [scanOffset] plus the [extraOffset]. For example, if the current |
| 214 * scanOffset is 10, then [appendSubstringToken(5, -1)] will append the | 215 * scanOffset is 10, then [appendSubstringToken(5, -1)] will append the |
| 215 * substring string [5,9). | 216 * substring string [5,9). |
| 216 * | 217 * |
| 217 * Note that [extraOffset] can only be used if the covered character(s) are | 218 * Note that [extraOffset] can only be used if the covered character(s) are |
| 218 * known to be ASCII. | 219 * known to be ASCII. |
| 219 */ | 220 */ |
| 220 DartDocToken createDartDocToken( | 221 DartDocToken createDartDocToken(TokenType info, int start, bool asciiOnly, |
| 221 PrecedenceInfo info, int start, bool asciiOnly, | |
| 222 [int extraOffset = 0]); | 222 [int extraOffset = 0]); |
| 223 | 223 |
| 224 /** Documentation in subclass [ArrayBasedScanner]. */ | 224 /** Documentation in subclass [ArrayBasedScanner]. */ |
| 225 void discardOpenLt(); | 225 void discardOpenLt(); |
| 226 | 226 |
| 227 /// Return true when at EOF. | 227 /// Return true when at EOF. |
| 228 bool atEndOfFile(); | 228 bool atEndOfFile(); |
| 229 | 229 |
| 230 Token tokenize() { | 230 Token tokenize() { |
| 231 while (!atEndOfFile()) { | 231 while (!atEndOfFile()) { |
| (...skipping 586 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 818 if (next > 127) { | 818 if (next > 127) { |
| 819 asciiOnlyLines = false; | 819 asciiOnlyLines = false; |
| 820 asciiOnlyComment = false; | 820 asciiOnlyComment = false; |
| 821 } | 821 } |
| 822 next = advance(); | 822 next = advance(); |
| 823 } | 823 } |
| 824 } | 824 } |
| 825 return next; | 825 return next; |
| 826 } | 826 } |
| 827 | 827 |
| 828 void appendComment(int start, PrecedenceInfo info, bool asciiOnly) { | 828 void appendComment(int start, TokenType info, bool asciiOnly) { |
| 829 if (!includeComments) return; | 829 if (!includeComments) return; |
| 830 CommentToken newComment = createCommentToken(info, start, asciiOnly); | 830 CommentToken newComment = createCommentToken(info, start, asciiOnly); |
| 831 if (scanGenericMethodComments) { | 831 if (scanGenericMethodComments) { |
| 832 String value = newComment.lexeme; | 832 String value = newComment.lexeme; |
| 833 int length = value.length; | 833 int length = value.length; |
| 834 if (length > 5 && | 834 if (length > 5 && |
| 835 value.codeUnitAt(0) == $SLASH && | 835 value.codeUnitAt(0) == $SLASH && |
| 836 value.codeUnitAt(1) == $STAR && | 836 value.codeUnitAt(1) == $STAR && |
| 837 value.codeUnitAt(2) == $EQ) { | 837 value.codeUnitAt(2) == $EQ) { |
| 838 newComment = new CommentToken.fromString( | 838 newComment = new CommentToken.fromString( |
| 839 GENERIC_METHOD_TYPE_ASSIGN, value, start); | 839 GENERIC_METHOD_TYPE_ASSIGN, value, start); |
| 840 } else if (length > 6 && | 840 } else if (length > 6 && |
| 841 value.codeUnitAt(0) == $SLASH && | 841 value.codeUnitAt(0) == $SLASH && |
| 842 value.codeUnitAt(1) == $STAR && | 842 value.codeUnitAt(1) == $STAR && |
| 843 value.codeUnitAt(2) == $LT && | 843 value.codeUnitAt(2) == $LT && |
| 844 value.codeUnitAt(length - 1) == $SLASH && | 844 value.codeUnitAt(length - 1) == $SLASH && |
| 845 value.codeUnitAt(length - 2) == $STAR && | 845 value.codeUnitAt(length - 2) == $STAR && |
| 846 value.codeUnitAt(length - 3) == $GT) { | 846 value.codeUnitAt(length - 3) == $GT) { |
| 847 newComment = | 847 newComment = |
| 848 new CommentToken.fromString(GENERIC_METHOD_TYPE_LIST, value, start); | 848 new CommentToken.fromString(GENERIC_METHOD_TYPE_LIST, value, start); |
| 849 } | 849 } |
| 850 } | 850 } |
| 851 _appendToCommentStream(newComment); | 851 _appendToCommentStream(newComment); |
| 852 } | 852 } |
| 853 | 853 |
| 854 void appendDartDoc(int start, PrecedenceInfo info, bool asciiOnly) { | 854 void appendDartDoc(int start, TokenType info, bool asciiOnly) { |
| 855 if (!includeComments) return; | 855 if (!includeComments) return; |
| 856 Token newComment = createDartDocToken(info, start, asciiOnly); | 856 Token newComment = createDartDocToken(info, start, asciiOnly); |
| 857 _appendToCommentStream(newComment); | 857 _appendToCommentStream(newComment); |
| 858 } | 858 } |
| 859 | 859 |
| 860 /** | 860 /** |
| 861 * Append the given token to the [tail] of the current stream of tokens. | 861 * Append the given token to the [tail] of the current stream of tokens. |
| 862 */ | 862 */ |
| 863 void appendToken(Token token) { | 863 void appendToken(Token token) { |
| 864 tail.next = token; | 864 tail.next = token; |
| (...skipping 333 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1198 int advanceAfterError(bool shouldAdvance) { | 1198 int advanceAfterError(bool shouldAdvance) { |
| 1199 if (atEndOfFile()) return $EOF; | 1199 if (atEndOfFile()) return $EOF; |
| 1200 if (shouldAdvance) { | 1200 if (shouldAdvance) { |
| 1201 return advance(); // Ensure progress. | 1201 return advance(); // Ensure progress. |
| 1202 } else { | 1202 } else { |
| 1203 return -1; | 1203 return -1; |
| 1204 } | 1204 } |
| 1205 } | 1205 } |
| 1206 } | 1206 } |
| 1207 | 1207 |
| 1208 PrecedenceInfo closeBraceInfoFor(BeginGroupToken begin) { | 1208 TokenType closeBraceInfoFor(BeginGroupToken begin) { |
| 1209 return const { | 1209 return const { |
| 1210 '(': CLOSE_PAREN_INFO, | 1210 '(': CLOSE_PAREN_INFO, |
| 1211 '[': CLOSE_SQUARE_BRACKET_INFO, | 1211 '[': CLOSE_SQUARE_BRACKET_INFO, |
| 1212 '{': CLOSE_CURLY_BRACKET_INFO, | 1212 '{': CLOSE_CURLY_BRACKET_INFO, |
| 1213 '<': GT_INFO, | 1213 '<': GT_INFO, |
| 1214 r'${': CLOSE_CURLY_BRACKET_INFO, | 1214 r'${': CLOSE_CURLY_BRACKET_INFO, |
| 1215 }[begin.lexeme]; | 1215 }[begin.lexeme]; |
| 1216 } | 1216 } |
| 1217 | 1217 |
| 1218 class LineStarts extends Object with ListMixin<int> { | 1218 class LineStarts extends Object with ListMixin<int> { |
| (...skipping 65 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1284 switchToUint32(newLength); | 1284 switchToUint32(newLength); |
| 1285 } | 1285 } |
| 1286 } | 1286 } |
| 1287 | 1287 |
| 1288 void switchToUint32(int newLength) { | 1288 void switchToUint32(int newLength) { |
| 1289 final newArray = new Uint32List(newLength); | 1289 final newArray = new Uint32List(newLength); |
| 1290 newArray.setRange(0, arrayLength, array); | 1290 newArray.setRange(0, arrayLength, array); |
| 1291 array = newArray; | 1291 array = newArray; |
| 1292 } | 1292 } |
| 1293 } | 1293 } |
| OLD | NEW |