Add some support for the code-point code-unit distinction.

runtime/vm/scanner.cc

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #include "vm/scanner.h"
 
 #include "platform/assert.h"
 #include "vm/flags.h"
 #include "vm/object.h"
 #include "vm/object_store.h"
@@ skipping 144 matching lines @@
 
 bool Scanner::IsIdentChar(int32_t c) {
   return IsLetter(c) || IsDecimalDigit(c) || (c == '_') || (c == '$');
 }
 
 
 bool Scanner::IsIdent(const String& str) {
   if (!str.IsOneByteString()) {
     return false;
   }
-  if (str.Length() == 0 || !IsIdentStartChar(str.CharAt(0))) {
+  if (str.Length() == 0 || !IsIdentStartChar(str.CodeUnitAt(0))) {
     return false;
   }
   for (int i =  1; i < str.Length(); i++) {
-    if (!IsIdentChar(str.CharAt(i))) {
+    if (!IsIdentChar(str.CodeUnitAt(i))) {
       return false;
     }
   }
   return true;
 }
 
 
 void Scanner::ReadChar() {
   if (lookahead_pos_ < source_length_) {
     if (c0_ == '\n') {
       newline_seen_ = true;
       c0_pos_.line++;
       c0_pos_.column = 0;
-      if (source_.CharAt(lookahead_pos_) == '\r') {
+      if (source_.CodeUnitAt(lookahead_pos_) == '\r') {
         // Replace a sequence of '\r' '\n' with a single '\n'.
         if (LookaheadChar(1) == '\n') {
           lookahead_pos_++;
         }
       }
     }
     lookahead_pos_++;
     c0_pos_.column++;
     c0_ = LookaheadChar(0);
     // Replace '\r' with '\n'.
     if (c0_ == '\r') {
       c0_ = '\n';
     }
   }
 }
 
 
 // Look ahead 'how_many' characters. Returns the character, or '\0' if
 // the lookahead position is beyond the end of the string. Does not
 // normalize line end characters into '\n'.
 int32_t Scanner::LookaheadChar(int how_many) {
   ASSERT(how_many >= 0);
   int32_t lookahead_char = '\0';
   if (lookahead_pos_ + how_many < source_length_) {
-    lookahead_char = source_.CharAt(lookahead_pos_ + how_many);
+    lookahead_char = source_.CodeUnitAt(lookahead_pos_ + how_many);
   }
   return lookahead_char;
 }
 
 
 void Scanner::ConsumeWhiteSpace() {
   while (c0_ == ' ' || c0_ == '\t' || c0_ == '\n') {
     ReadChar();
   }
 }
@@ skipping 34 matching lines @@
   current_token_.kind =
     (nesting_level == 0) ? Token::kWHITESP : Token::kILLEGAL;
 }
 
 
 void Scanner::ScanIdentChars(bool allow_dollar) {
   ASSERT(IsIdentStartChar(c0_));
   ASSERT(allow_dollar || (c0_ != '$'));
   int ident_length = 0;
   int ident_pos = lookahead_pos_;
-  int32_t ident_char0 = source_.CharAt(ident_pos);
+  int32_t ident_char0 = source_.CodeUnitAt(ident_pos);
   while (IsIdentChar(c0_) && (allow_dollar || (c0_ != '$'))) {
     ReadChar();
     ident_length++;
   }
 
   // Check whether the characters we read are a known keyword.
   // Note, can't use strcmp since token_chars is not null-terminated.
   int i = 0;
   while (i < Token::numKeywords &&
          keywords_[i].keyword_chars[0] <= ident_char0) {
     if (keywords_[i].keyword_len == ident_length) {
       const char* keyword = keywords_[i].keyword_chars;
       int char_pos = 0;
       while ((char_pos < ident_length) &&
-             (keyword[char_pos] == source_.CharAt(ident_pos + char_pos))) {
+             (static_cast<uint32_t>(keyword[char_pos]) ==
+              source_.CodeUnitAt(ident_pos + char_pos))) {
         char_pos++;
       }
       if (char_pos == ident_length) {
         if (keywords_[i].keyword_symbol == NULL) {
           String& symbol = String::ZoneHandle();
           symbol ^= keyword_symbol_table_.At(i);
           ASSERT(!symbol.IsNull());
           keywords_[i].keyword_symbol = &symbol;
         }
         current_token_.literal = keywords_[i].keyword_symbol;
@@ skipping 663 matching lines @@
               "%c%#"Px"", kPrivateKeySeparator, key_value);
   const String& result = String::Handle(String::New(private_key, Heap::kOld));
   return result.raw();
 }
 
 
 void Scanner::InitOnce() {
 }
 
 }  // namespace dart