Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(122)

Side by Side Diff: sdk/lib/_internal/compiler/implementation/scanner/token.dart

Issue 11368138: Add some support for the code-point code-unit distinction. (Closed) Base URL: https://dart.googlecode.com/svn/branches/bleeding_edge/dart
Patch Set: New version integrates feedback, adds less to standard String class. Created 8 years, 1 month ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
OLDNEW
1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file 1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file
2 // for details. All rights reserved. Use of this source code is governed by a 2 // for details. All rights reserved. Use of this source code is governed by a
3 // BSD-style license that can be found in the LICENSE file. 3 // BSD-style license that can be found in the LICENSE file.
4 4
5 part of scanner; 5 part of scanner;
6 6
7 const int EOF_TOKEN = 0; 7 const int EOF_TOKEN = 0;
8 8
9 const int KEYWORD_TOKEN = $k; 9 const int KEYWORD_TOKEN = $k;
10 const int IDENTIFIER_TOKEN = $a; 10 const int IDENTIFIER_TOKEN = $a;
(...skipping 213 matching lines...) Expand 10 before | Expand all | Expand 10 after
224 assert(initial + terminal <= stringValue.length); 224 assert(initial + terminal <= stringValue.length);
225 return new StringWrapper( 225 return new StringWrapper(
226 stringValue.substring(initial, stringValue.length - terminal)); 226 stringValue.substring(initial, stringValue.length - terminal));
227 } 227 }
228 228
229 bool get isEmpty => stringValue.isEmpty; 229 bool get isEmpty => stringValue.isEmpty;
230 230
231 bool isPrivate() => !isEmpty && identical(stringValue.charCodeAt(0), $_); 231 bool isPrivate() => !isEmpty && identical(stringValue.charCodeAt(0), $_);
232 } 232 }
233 233
234
235 // TODO(erikcorry): Use the new code point iterator on String when it is
236 // available.
234 class StringCodeIterator implements Iterator<int> { 237 class StringCodeIterator implements Iterator<int> {
235 final String string; 238 final String string;
236 int index; 239 int index;
237 final int end; 240 final int end;
238 241
239 StringCodeIterator(String string) : 242 StringCodeIterator(String string) :
240 this.string = string, index = 0, end = string.length; 243 this.string = string, index = 0, end = string.length;
241 244
242 StringCodeIterator.substring(this.string, this.index, this.end) { 245 StringCodeIterator.substring(this.string, this.index, this.end) {
243 assert(0 <= index); 246 assert(0 <= index);
244 assert(index <= end); 247 assert(index <= end);
245 assert(end <= string.length); 248 assert(end <= string.length);
246 } 249 }
247 250
248 bool get hasNext => index < end; 251 bool get hasNext => index < end;
249 int next() => string.charCodeAt(index++); 252 int next() {
253 int charCode = string.charCodeAt(index++);
254 // Skip trail surrogate.
255 if (charCode >= String.SMP_CODE_POINT_BASE) index++;
256 return charCode;
257 }
250 } 258 }
251 259
252 class BeginGroupToken extends StringToken { 260 class BeginGroupToken extends StringToken {
253 Token endGroup; 261 Token endGroup;
254 BeginGroupToken(PrecedenceInfo info, String value, int charOffset) 262 BeginGroupToken(PrecedenceInfo info, String value, int charOffset)
255 : super(info, value, charOffset); 263 : super(info, value, charOffset);
256 } 264 }
257 265
258 bool isUserDefinableOperator(String value) { 266 bool isUserDefinableOperator(String value) {
259 return 267 return
(...skipping 235 matching lines...) Expand 10 before | Expand all | Expand 10 after
495 503
496 const PrecedenceInfo HEXADECIMAL_INFO = 504 const PrecedenceInfo HEXADECIMAL_INFO =
497 const PrecedenceInfo(const SourceString('hexadecimal'), 0, HEXADECIMAL_TOKEN); 505 const PrecedenceInfo(const SourceString('hexadecimal'), 0, HEXADECIMAL_TOKEN);
498 506
499 const PrecedenceInfo COMMENT_INFO = 507 const PrecedenceInfo COMMENT_INFO =
500 const PrecedenceInfo(const SourceString('comment'), 0, COMMENT_TOKEN); 508 const PrecedenceInfo(const SourceString('comment'), 0, COMMENT_TOKEN);
501 509
502 // For reporting lexical errors. 510 // For reporting lexical errors.
503 const PrecedenceInfo ERROR_INFO = 511 const PrecedenceInfo ERROR_INFO =
504 const PrecedenceInfo(const SourceString('?'), 0, UNKNOWN_TOKEN); 512 const PrecedenceInfo(const SourceString('?'), 0, UNKNOWN_TOKEN);
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698