Add some support for the code-point code-unit distinction.

runtime/vm/unicode.h

 // Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #ifndef VM_UNICODE_H_
 #define VM_UNICODE_H_
 
 #include "vm/allocation.h"
 #include "vm/globals.h"
 
 namespace dart {
 
 class String;
 
+class Utf16 : AllStatic {
+ public:
+  static const int32_t kMaxCodeUnit = 0xFFFF;
+  static const int32_t kMaxCodePoint = 0x10FFFF;
+
+  static bool IsLeadSurrogate(int32_t c) {
+    return c >= kLeadBase && c < kLeadEnd;
+  }
+
+  static bool IsTrailSurrogate(int32_t c) {
+    return c >= kTrailBase && c < kTrailEnd;
+  }
+
+  static bool IsSurrogate(int32_t c) {
+    return (c & 0xFFFFF800u) == 0xD800u;
+  }
+
+  static int32_t CodePointFromCodeUnits(int32_t lead, int32_t trail) {

[Søren Gjesse, 2012/11/19 14:18:36]

Assert that lead is lead surrogare and that train is a trail surrogate.

+    return kSurrogateEncodingBase +
+           ((lead & kSurrogateMask) << 10) + (trail & kSurrogateMask);
+  }
+
+  static int32_t LeadFromCodePoint(int32_t code_point) {
+    ASSERT(code_point >= kSurrogateEncodingBase);
+    return kLeadBase +
+           (((code_point - kSurrogateEncodingBase) >> 10) & kSurrogateMask);
+  }
+
+  static int32_t TrailFromCodePoint(int32_t code_point) {
+    ASSERT(code_point >= kSurrogateEncodingBase);
+    return kTrailBase + (code_point & kSurrogateMask);
+  }
+
+  // Gets the 21 bit Unicode code point at the given index in a string.  If the
+  // returned value is greater than kMaxCodePoint then the next position of the
+  // string encodes a trail surrogate and should be skipped on iteration.  May
+  // return individual surrogate values if they are not part of a pair.
+  static int32_t CodePointAt(const String& str, int index);
+
+ private:
+  static const int32_t kLeadBase = 0xD800;
+  static const int32_t kLeadEnd = 0xDBFF;
+  static const int32_t kTrailBase = 0xDC00;
+  static const int32_t kTrailEnd = 0xDFFF;
+  static const int32_t kSurrogateMask = 0x3FF;
+  static const int32_t kSurrogateEncodingBase = 0x10000;
+};
+
+
 class Utf8 : AllStatic {
  public:
   enum Type {
     kLatin1 = 0,  // Latin-1 character set.
     kBMP,  // Basic Multilingual Plane.
     kSMP,  // Supplementary Multilingual Plane.
   };
 
   static const intptr_t kMaxOneByteChar   = 0x7F;
   static const intptr_t kMaxTwoByteChar   = 0x7FF;
   static const intptr_t kMaxThreeByteChar = 0xFFFF;
   static const intptr_t kMaxFourByteChar  = 0x10FFFF;
-  static const intptr_t kMaxBmpCodepoint  = 0xffff;
-  static const int32_t kLeadOffset = (0xD800 - (0x10000 >> 10));
-  static const int32_t kSurrogateOffset = (0x10000 - (0xD800 << 10) - 0xDC00);
 
-  static void ConvertUTF32ToUTF16(int32_t codepoint, uint16_t* dst);
-  static intptr_t CodePointCount(const uint8_t* utf8_array,
-                                 intptr_t array_len,
-                                 Type* type);
+  static const int32_t kInvalidCodePoint = -1;
+
+  static intptr_t CodeUnitCount(const uint8_t* utf8_array,
+                                intptr_t array_len,
+                                Type* type);
 
   // Returns true if 'utf8_array' is a valid UTF-8 string.
   static bool IsValid(const uint8_t* utf8_array, intptr_t array_len);
 
   static intptr_t Length(int32_t ch);
   static intptr_t Length(const String& str);
 
   static intptr_t Encode(int32_t ch, char* dst);
   static intptr_t Encode(const String& src, char* dst, intptr_t len);
 
   static intptr_t Decode(const uint8_t* utf8_array,
                          intptr_t array_len,
                          int32_t* ch);
 
   static bool DecodeToLatin1(const uint8_t* utf8_array,
                              intptr_t array_len,
                              uint8_t* dst,
                              intptr_t len);
   static bool DecodeToUTF16(const uint8_t* utf8_array,
                             intptr_t array_len,
                             uint16_t* dst,
                             intptr_t len);
   static bool DecodeToUTF32(const uint8_t* utf8_array,
                             intptr_t array_len,
-                            uint32_t* dst,
+                            int32_t* dst,
                             intptr_t len);
   static bool DecodeCStringToUTF32(const char* str,
-                                   uint32_t* dst,
+                                   int32_t* dst,
                                    intptr_t len) {
     ASSERT(str != NULL);
     intptr_t array_len = strlen(str);
     const uint8_t* utf8_array = reinterpret_cast<const uint8_t*>(str);
     return DecodeToUTF32(utf8_array, array_len, dst, len);
   }
 };
 
 
 class CaseMapping : AllStatic {
@@ skipping 55 matching lines @@
   // Data for small code points with one mapping
   static const int16_t stage2_[];
 
   // Data for large code points or code points with both mappings.
   static const int32_t stage2_exception_[][2];
 };
 
 }  // namespace dart
 
 #endif  // VM_UNICODE_H_