OLD | NEW |
1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 part of scanner; | 5 part of scanner; |
6 | 6 |
7 const int EOF_TOKEN = 0; | 7 const int EOF_TOKEN = 0; |
8 | 8 |
9 const int KEYWORD_TOKEN = $k; | 9 const int KEYWORD_TOKEN = $k; |
10 const int IDENTIFIER_TOKEN = $a; | 10 const int IDENTIFIER_TOKEN = $a; |
(...skipping 213 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
224 assert(initial + terminal <= stringValue.length); | 224 assert(initial + terminal <= stringValue.length); |
225 return new StringWrapper( | 225 return new StringWrapper( |
226 stringValue.substring(initial, stringValue.length - terminal)); | 226 stringValue.substring(initial, stringValue.length - terminal)); |
227 } | 227 } |
228 | 228 |
229 bool get isEmpty => stringValue.isEmpty; | 229 bool get isEmpty => stringValue.isEmpty; |
230 | 230 |
231 bool isPrivate() => !isEmpty && identical(stringValue.charCodeAt(0), $_); | 231 bool isPrivate() => !isEmpty && identical(stringValue.charCodeAt(0), $_); |
232 } | 232 } |
233 | 233 |
| 234 |
| 235 // TODO(erikcorry): Use the new code point iterator on String when it is |
| 236 // available. |
234 class StringCodeIterator implements Iterator<int> { | 237 class StringCodeIterator implements Iterator<int> { |
235 final String string; | 238 final String string; |
236 int index; | 239 int index; |
237 final int end; | 240 final int end; |
238 | 241 |
239 StringCodeIterator(String string) : | 242 StringCodeIterator(String string) : |
240 this.string = string, index = 0, end = string.length; | 243 this.string = string, index = 0, end = string.length; |
241 | 244 |
242 StringCodeIterator.substring(this.string, this.index, this.end) { | 245 StringCodeIterator.substring(this.string, this.index, this.end) { |
243 assert(0 <= index); | 246 assert(0 <= index); |
244 assert(index <= end); | 247 assert(index <= end); |
245 assert(end <= string.length); | 248 assert(end <= string.length); |
246 } | 249 } |
247 | 250 |
248 bool get hasNext => index < end; | 251 bool get hasNext => index < end; |
249 int next() => string.charCodeAt(index++); | 252 int next() { |
| 253 int charCode = string.charCodeAt(index++); |
| 254 // Skip trail surrogate. |
| 255 if (charCode >= String.SMP_CODE_POINT_BASE) index++; |
| 256 return charCode; |
| 257 } |
250 } | 258 } |
251 | 259 |
252 class BeginGroupToken extends StringToken { | 260 class BeginGroupToken extends StringToken { |
253 Token endGroup; | 261 Token endGroup; |
254 BeginGroupToken(PrecedenceInfo info, String value, int charOffset) | 262 BeginGroupToken(PrecedenceInfo info, String value, int charOffset) |
255 : super(info, value, charOffset); | 263 : super(info, value, charOffset); |
256 } | 264 } |
257 | 265 |
258 bool isUserDefinableOperator(String value) { | 266 bool isUserDefinableOperator(String value) { |
259 return | 267 return |
(...skipping 235 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
495 | 503 |
496 const PrecedenceInfo HEXADECIMAL_INFO = | 504 const PrecedenceInfo HEXADECIMAL_INFO = |
497 const PrecedenceInfo(const SourceString('hexadecimal'), 0, HEXADECIMAL_TOKEN); | 505 const PrecedenceInfo(const SourceString('hexadecimal'), 0, HEXADECIMAL_TOKEN); |
498 | 506 |
499 const PrecedenceInfo COMMENT_INFO = | 507 const PrecedenceInfo COMMENT_INFO = |
500 const PrecedenceInfo(const SourceString('comment'), 0, COMMENT_TOKEN); | 508 const PrecedenceInfo(const SourceString('comment'), 0, COMMENT_TOKEN); |
501 | 509 |
502 // For reporting lexical errors. | 510 // For reporting lexical errors. |
503 const PrecedenceInfo ERROR_INFO = | 511 const PrecedenceInfo ERROR_INFO = |
504 const PrecedenceInfo(const SourceString('?'), 0, UNKNOWN_TOKEN); | 512 const PrecedenceInfo(const SourceString('?'), 0, UNKNOWN_TOKEN); |
OLD | NEW |