OLD | NEW |
1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 library fasta.scanner.abstract_scanner; | 5 library fasta.scanner.abstract_scanner; |
6 | 6 |
| 7 import 'dart:collection' show |
| 8 ListMixin; |
| 9 |
| 10 import 'dart:typed_data' show |
| 11 Uint16List, |
| 12 Uint32List; |
| 13 |
7 import '../scanner.dart' show | 14 import '../scanner.dart' show |
8 ErrorToken, | 15 ErrorToken, |
9 Scanner, | 16 Scanner, |
10 buildUnexpectedCharacterToken; | 17 buildUnexpectedCharacterToken; |
11 | 18 |
12 import 'error_token.dart' show | 19 import 'error_token.dart' show |
13 UnmatchedToken, | 20 UnmatchedToken, |
14 UnterminatedToken; | 21 UnterminatedToken; |
15 | 22 |
16 import 'keyword.dart' show | 23 import 'keyword.dart' show |
(...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
48 * is not exposed to clients of the scanner, which are expected to invoke | 55 * is not exposed to clients of the scanner, which are expected to invoke |
49 * [firstToken] to access the token stream. | 56 * [firstToken] to access the token stream. |
50 */ | 57 */ |
51 final Token tokens = new SymbolToken(EOF_INFO, -1); | 58 final Token tokens = new SymbolToken(EOF_INFO, -1); |
52 | 59 |
53 /** | 60 /** |
54 * A pointer to the last scanned token. | 61 * A pointer to the last scanned token. |
55 */ | 62 */ |
56 Token tail; | 63 Token tail; |
57 | 64 |
58 final List<int> lineStarts = <int>[0]; | 65 final List<int> lineStarts; |
59 | 66 |
60 AbstractScanner(this.includeComments) { | 67 AbstractScanner(this.includeComments, {int numberOfBytesHint}) |
| 68 : lineStarts = new LineStarts(numberOfBytesHint) { |
61 this.tail = this.tokens; | 69 this.tail = this.tokens; |
62 } | 70 } |
63 | 71 |
64 /** | 72 /** |
65 * Advances and returns the next character. | 73 * Advances and returns the next character. |
66 * | 74 * |
67 * If the next character is non-ASCII, then the returned value depends on the | 75 * If the next character is non-ASCII, then the returned value depends on the |
68 * scanner implementation. The [Utf8BytesScanner] returns a UTF-8 byte, while | 76 * scanner implementation. The [Utf8BytesScanner] returns a UTF-8 byte, while |
69 * the [StringScanner] returns a UTF-16 code unit. | 77 * the [StringScanner] returns a UTF-16 code unit. |
70 * | 78 * |
(...skipping 1087 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1158 | 1166 |
1159 PrecedenceInfo closeBraceInfoFor(BeginGroupToken begin) { | 1167 PrecedenceInfo closeBraceInfoFor(BeginGroupToken begin) { |
1160 return const { | 1168 return const { |
1161 '(': CLOSE_PAREN_INFO, | 1169 '(': CLOSE_PAREN_INFO, |
1162 '[': CLOSE_SQUARE_BRACKET_INFO, | 1170 '[': CLOSE_SQUARE_BRACKET_INFO, |
1163 '{': CLOSE_CURLY_BRACKET_INFO, | 1171 '{': CLOSE_CURLY_BRACKET_INFO, |
1164 '<': GT_INFO, | 1172 '<': GT_INFO, |
1165 r'${': CLOSE_CURLY_BRACKET_INFO, | 1173 r'${': CLOSE_CURLY_BRACKET_INFO, |
1166 }[begin.value]; | 1174 }[begin.value]; |
1167 } | 1175 } |
| 1176 |
| 1177 class LineStarts extends Object with ListMixin<int> { |
| 1178 List<int> array; |
| 1179 int arrayLength = 0; |
| 1180 |
| 1181 LineStarts(int numberOfBytesHint) { |
| 1182 // Let's assume the average Dart file is 300 bytes. |
| 1183 if (numberOfBytesHint == null) numberOfBytesHint = 300; |
| 1184 |
| 1185 // Let's assume we have on average 22 bytes per line. |
| 1186 final int expectedNumberOfLines = 1 + (numberOfBytesHint ~/ 22); |
| 1187 |
| 1188 if (numberOfBytesHint > 65535) { |
| 1189 array = new Uint32List(expectedNumberOfLines); |
| 1190 } else { |
| 1191 array = new Uint16List(expectedNumberOfLines); |
| 1192 } |
| 1193 |
| 1194 // The first line starts at character offset 0. |
| 1195 add(0); |
| 1196 } |
| 1197 |
| 1198 // Implement abstract members used by [ListMixin] |
| 1199 |
| 1200 int get length => arrayLength; |
| 1201 |
| 1202 int operator[](int index) { |
| 1203 assert(index < arrayLength); |
| 1204 return array[index]; |
| 1205 } |
| 1206 |
| 1207 void set length(int newLength) { |
| 1208 if (newLength > array.length) { |
| 1209 grow(newLength); |
| 1210 } |
| 1211 arrayLength = newLength; |
| 1212 } |
| 1213 |
| 1214 void operator[]=(int index, int value) { |
| 1215 if (value > 65535 && array is! Uint32List) { |
| 1216 switchToUint32(array.length); |
| 1217 } |
| 1218 array[index] = value; |
| 1219 } |
| 1220 |
| 1221 // Specialize methods from [ListMixin]. |
| 1222 void add(int value) { |
| 1223 if (arrayLength >= array.length) { |
| 1224 grow(0); |
| 1225 } |
| 1226 if (value > 65535 && array is! Uint32List) { |
| 1227 switchToUint32(array.length); |
| 1228 } |
| 1229 array[arrayLength++] = value; |
| 1230 } |
| 1231 |
| 1232 // Helper methods. |
| 1233 |
| 1234 void grow(int newLengthMinimum) { |
| 1235 int newLength = array.length * 2; |
| 1236 if (newLength < newLengthMinimum) newLength = newLengthMinimum; |
| 1237 |
| 1238 if (array is Uint16List) { |
| 1239 final newArray = new Uint16List(newLength); |
| 1240 newArray.setRange(0, arrayLength, array); |
| 1241 array = newArray; |
| 1242 } else { |
| 1243 switchToUint32(newLength); |
| 1244 } |
| 1245 } |
| 1246 |
| 1247 void switchToUint32(int newLength) { |
| 1248 final newArray = new Uint32List(newLength); |
| 1249 newArray.setRange(0, arrayLength, array); |
| 1250 array = newArray; |
| 1251 } |
| 1252 } |
OLD | NEW |