Add some support for the code-point code-unit distinction.

sdk/lib/_internal/compiler/implementation/scanner/token.dart

 // Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 part of scanner;
 
 const int EOF_TOKEN = 0;
 
 const int KEYWORD_TOKEN = $k;
 const int IDENTIFIER_TOKEN = $a;
@@ skipping 228 matching lines @@
   StringCodeIterator(String string) :
     this.string = string, index = 0, end = string.length;
 
   StringCodeIterator.substring(this.string, this.index, this.end) {
     assert(0 <= index);
     assert(index <= end);
     assert(end <= string.length);
   }
 
   bool get hasNext => index < end;
-  int next() => string.charCodeAt(index++);
+  int next() {

[floitsch, 2012/11/08 15:28:21]

maybe add TODO to switch to charcode-iterator once we have one.

[erikcorry, 2012/11/15 13:28:25]

Done.

+    int charCode = string.charCodeAt(index++);
+    if (charCode >= 0x10000) index++;  // Skip trail surrogate.
+    return charCode;
+  }
 }
 
 class BeginGroupToken extends StringToken {
   Token endGroup;
   BeginGroupToken(PrecedenceInfo info, String value, int charOffset)
     : super(info, value, charOffset);
 }
 
 bool isUserDefinableOperator(String value) {
   return
@@ skipping 235 matching lines @@
 
 const PrecedenceInfo HEXADECIMAL_INFO =
   const PrecedenceInfo(const SourceString('hexadecimal'), 0, HEXADECIMAL_TOKEN);
 
 const PrecedenceInfo COMMENT_INFO =
   const PrecedenceInfo(const SourceString('comment'), 0, COMMENT_TOKEN);
 
 // For reporting lexical errors.
 const PrecedenceInfo ERROR_INFO =
   const PrecedenceInfo(const SourceString('?'), 0, UNKNOWN_TOKEN);