| OLD | NEW |
| 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 library fasta.scanner.abstract_scanner; | 5 library fasta.scanner.abstract_scanner; |
| 6 | 6 |
| 7 import 'dart:collection' show |
| 8 ListMixin; |
| 9 |
| 10 import 'dart:typed_data' show |
| 11 Uint16List, |
| 12 Uint32List; |
| 13 |
| 7 import '../scanner.dart' show | 14 import '../scanner.dart' show |
| 8 ErrorToken, | 15 ErrorToken, |
| 9 Scanner, | 16 Scanner, |
| 10 buildUnexpectedCharacterToken; | 17 buildUnexpectedCharacterToken; |
| 11 | 18 |
| 12 import 'error_token.dart' show | 19 import 'error_token.dart' show |
| 13 UnmatchedToken, | 20 UnmatchedToken, |
| 14 UnterminatedToken; | 21 UnterminatedToken; |
| 15 | 22 |
| 16 import 'keyword.dart' show | 23 import 'keyword.dart' show |
| (...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 48 * is not exposed to clients of the scanner, which are expected to invoke | 55 * is not exposed to clients of the scanner, which are expected to invoke |
| 49 * [firstToken] to access the token stream. | 56 * [firstToken] to access the token stream. |
| 50 */ | 57 */ |
| 51 final Token tokens = new SymbolToken(EOF_INFO, -1); | 58 final Token tokens = new SymbolToken(EOF_INFO, -1); |
| 52 | 59 |
| 53 /** | 60 /** |
| 54 * A pointer to the last scanned token. | 61 * A pointer to the last scanned token. |
| 55 */ | 62 */ |
| 56 Token tail; | 63 Token tail; |
| 57 | 64 |
| 58 final List<int> lineStarts = <int>[0]; | 65 final List<int> lineStarts; |
| 59 | 66 |
| 60 AbstractScanner(this.includeComments) { | 67 AbstractScanner(this.includeComments, {int numberOfBytesHint}) |
| 68 : lineStarts = new LineStarts(numberOfBytesHint) { |
| 61 this.tail = this.tokens; | 69 this.tail = this.tokens; |
| 62 } | 70 } |
| 63 | 71 |
| 64 /** | 72 /** |
| 65 * Advances and returns the next character. | 73 * Advances and returns the next character. |
| 66 * | 74 * |
| 67 * If the next character is non-ASCII, then the returned value depends on the | 75 * If the next character is non-ASCII, then the returned value depends on the |
| 68 * scanner implementation. The [Utf8BytesScanner] returns a UTF-8 byte, while | 76 * scanner implementation. The [Utf8BytesScanner] returns a UTF-8 byte, while |
| 69 * the [StringScanner] returns a UTF-16 code unit. | 77 * the [StringScanner] returns a UTF-16 code unit. |
| 70 * | 78 * |
| (...skipping 1087 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1158 | 1166 |
| 1159 PrecedenceInfo closeBraceInfoFor(BeginGroupToken begin) { | 1167 PrecedenceInfo closeBraceInfoFor(BeginGroupToken begin) { |
| 1160 return const { | 1168 return const { |
| 1161 '(': CLOSE_PAREN_INFO, | 1169 '(': CLOSE_PAREN_INFO, |
| 1162 '[': CLOSE_SQUARE_BRACKET_INFO, | 1170 '[': CLOSE_SQUARE_BRACKET_INFO, |
| 1163 '{': CLOSE_CURLY_BRACKET_INFO, | 1171 '{': CLOSE_CURLY_BRACKET_INFO, |
| 1164 '<': GT_INFO, | 1172 '<': GT_INFO, |
| 1165 r'${': CLOSE_CURLY_BRACKET_INFO, | 1173 r'${': CLOSE_CURLY_BRACKET_INFO, |
| 1166 }[begin.value]; | 1174 }[begin.value]; |
| 1167 } | 1175 } |
| 1176 |
| 1177 class LineStarts extends Object with ListMixin<int> { |
| 1178 List<int> array; |
| 1179 int arrayLength = 0; |
| 1180 |
| 1181 LineStarts(int numberOfBytesHint) { |
| 1182 // Let's assume the average Dart file is 300 bytes. |
| 1183 if (numberOfBytesHint == null) numberOfBytesHint = 300; |
| 1184 |
| 1185 // Let's assume we have on average 22 bytes per line. |
| 1186 final int expectedNumberOfLines = 1 + (numberOfBytesHint ~/ 22); |
| 1187 |
| 1188 if (numberOfBytesHint > 65535) { |
| 1189 array = new Uint32List(expectedNumberOfLines); |
| 1190 } else { |
| 1191 array = new Uint16List(expectedNumberOfLines); |
| 1192 } |
| 1193 |
| 1194 // The first line starts at character offset 0. |
| 1195 add(0); |
| 1196 } |
| 1197 |
| 1198 // Implement abstract members used by [ListMixin] |
| 1199 |
| 1200 int get length => arrayLength; |
| 1201 |
| 1202 int operator[](int index) { |
| 1203 assert(index < arrayLength); |
| 1204 return array[index]; |
| 1205 } |
| 1206 |
| 1207 void set length(int newLength) { |
| 1208 if (newLength > array.length) { |
| 1209 grow(newLength); |
| 1210 } |
| 1211 arrayLength = newLength; |
| 1212 } |
| 1213 |
| 1214 void operator[]=(int index, int value) { |
| 1215 if (value > 65535 && array is! Uint32List) { |
| 1216 switchToUint32(array.length); |
| 1217 } |
| 1218 array[index] = value; |
| 1219 } |
| 1220 |
| 1221 // Specialize methods from [ListMixin]. |
| 1222 void add(int value) { |
| 1223 if (arrayLength >= array.length) { |
| 1224 grow(0); |
| 1225 } |
| 1226 if (value > 65535 && array is! Uint32List) { |
| 1227 switchToUint32(array.length); |
| 1228 } |
| 1229 array[arrayLength++] = value; |
| 1230 } |
| 1231 |
| 1232 // Helper methods. |
| 1233 |
| 1234 void grow(int newLengthMinimum) { |
| 1235 int newLength = array.length * 2; |
| 1236 if (newLength < newLengthMinimum) newLength = newLengthMinimum; |
| 1237 |
| 1238 if (array is Uint16List) { |
| 1239 final newArray = new Uint16List(newLength); |
| 1240 newArray.setRange(0, arrayLength, array); |
| 1241 array = newArray; |
| 1242 } else { |
| 1243 switchToUint32(newLength); |
| 1244 } |
| 1245 } |
| 1246 |
| 1247 void switchToUint32(int newLength) { |
| 1248 final newArray = new Uint32List(newLength); |
| 1249 newArray.setRange(0, arrayLength, array); |
| 1250 array = newArray; |
| 1251 } |
| 1252 } |
| OLD | NEW |