Add some support for the code-point code-unit distinction.

sdk/lib/_internal/compiler/implementation/scanner/token.dart

 // Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 part of scanner;
 
 const int EOF_TOKEN = 0;
 
 const int KEYWORD_TOKEN = $k;
 const int IDENTIFIER_TOKEN = $a;
@@ skipping 213 matching lines @@
     assert(initial + terminal <= stringValue.length);
     return new StringWrapper(
         stringValue.substring(initial, stringValue.length - terminal));
   }
 
   bool get isEmpty => stringValue.isEmpty;
 
   bool isPrivate() => !isEmpty && identical(stringValue.charCodeAt(0), $_);
 }
 
+
+// TODO(erikcorry): Use the new code point iterator on String when it is
+// available.
 class StringCodeIterator implements Iterator<int> {
   final String string;
   int index;
   final int end;
 
   StringCodeIterator(String string) :
     this.string = string, index = 0, end = string.length;
 
   StringCodeIterator.substring(this.string, this.index, this.end) {
     assert(0 <= index);
     assert(index <= end);
     assert(end <= string.length);
   }
 
   bool get hasNext => index < end;
-  int next() => string.charCodeAt(index++);
+  int next() {
+    int charCode = string.charCodeAt(index++);
+    // Skip trail surrogate.
+    if (charCode >= String.SMP_CODE_POINT_BASE) index++;
+    return charCode;
+  }
 }
 
 class BeginGroupToken extends StringToken {
   Token endGroup;
   BeginGroupToken(PrecedenceInfo info, String value, int charOffset)
     : super(info, value, charOffset);
 }
 
 bool isUserDefinableOperator(String value) {
   return
@@ skipping 235 matching lines @@
 
 const PrecedenceInfo HEXADECIMAL_INFO =
   const PrecedenceInfo(const SourceString('hexadecimal'), 0, HEXADECIMAL_TOKEN);
 
 const PrecedenceInfo COMMENT_INFO =
   const PrecedenceInfo(const SourceString('comment'), 0, COMMENT_TOKEN);
 
 // For reporting lexical errors.
 const PrecedenceInfo ERROR_INFO =
   const PrecedenceInfo(const SourceString('?'), 0, UNKNOWN_TOKEN);