Revert "Add some support for the code-point code-unit distinction."

runtime/vm/unicode.cc

 // Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #include "vm/unicode.h"
 
 #include "vm/allocation.h"
 #include "vm/globals.h"
 #include "vm/object.h"
 
@@ skipping 30 matching lines @@
 };
 
 
 // Minimum values of code points used to check shortest form.
 static const uint32_t kOverlongMinimum[7] = {
   0,  // padding
   0x0,
   0x80,
   0x800,
   0x10000,
-  0xFFFFFFFF,  // We never allow 5 byte sequences.
-  0xFFFFFFFF   // We never allow 6 byte sequences.
+  0xFFFFFFFF,
+  0xFFFFFFFF
 };
 
 
 static bool IsTrailByte(uint8_t code_unit) {
   return (code_unit & 0xc0) == 0x80;
 }
 
 
 static bool IsLatin1SequenceStart(uint8_t code_unit) {
   // Check is codepoint is <= U+00FF
   return (code_unit <= Utf8::kMaxOneByteChar);
 }
 
 
 static bool IsSupplementarySequenceStart(uint8_t code_unit) {
-  // Check the UTF-8 code unit to determine if it is a sequence start for a
-  // code point >= U+10000.
+  // Check is codepoint is >= U+10000.
   return (code_unit >= 0xF0);
 }
 
 
 // Returns true if the code point value is above Plane 17.
-static bool IsOutOfRange(int32_t code_point) {
-  return (code_point > Utf16::kMaxCodePoint);
+static bool IsOutOfRange(uint32_t code_point) {
+  return (code_point > 0x10FFFF);
 }
 
 
 // Returns true if the byte sequence is ill-formed.
 static bool IsNonShortestForm(uint32_t code_point, size_t num_bytes) {
   return code_point < kOverlongMinimum[num_bytes];
 }
 
 
-// Returns a count of the number of UTF-16 code units represented by this UTF-8
-// array.  Type is kASCII for 7-bit-only.  If there are surrogate pairs then
-// the type is kSupplementary.  Otherwise it is kBMP.
-intptr_t Utf8::CodeUnitCount(const uint8_t* utf8_array,
-                             intptr_t array_len,
-                             Type* type) {
+// Returns a count of the number of UTF-8 trail bytes.
+intptr_t Utf8::CodePointCount(const uint8_t* utf8_array,
+                              intptr_t array_len,
+                              Type* type) {
   intptr_t len = 0;
   Type char_type = kLatin1;
   for (intptr_t i = 0; i < array_len; i++) {
     uint8_t code_unit = utf8_array[i];
     if (!IsTrailByte(code_unit)) {
       ++len;
     }
     if (!IsLatin1SequenceStart(code_unit)) {  // > U+00FF
       if (IsSupplementarySequenceStart(code_unit)) {  // >= U+10000
         char_type = kSupplementary;
-        ++len;  // Surrogate pair in the UTF-16 encoding.
+        ++len;
       } else if (char_type == kLatin1) {
         char_type = kBMP;
       }
     }
   }
   *type = char_type;
   return len;
 }
 
 
-// Returns true if str is a valid UTF-8 string.
+// Returns true if str is a valid NUL-terminated UTF-8 string.
 bool Utf8::IsValid(const uint8_t* utf8_array, intptr_t array_len) {
   intptr_t i = 0;
   while (i < array_len) {
     uint32_t ch = utf8_array[i] & 0xFF;
     intptr_t j = 1;
     if (ch >= 0x80) {
       int8_t num_trail_bytes = kTrailBytes[ch];
       bool is_malformed = false;
       for (; j < num_trail_bytes; ++j) {
         if ((i + j) < array_len) {
@@ skipping 81 matching lines @@
     Utf8::Encode(ch, &dst[pos]);
     pos += num_bytes;
   }
   return pos;
 }
 
 
 intptr_t Utf8::Decode(const uint8_t* utf8_array,
                       intptr_t array_len,
                       int32_t* dst) {
-  int32_t ch = utf8_array[0] & 0xFF;
+  uint32_t ch = utf8_array[0] & 0xFF;
   intptr_t i = 1;
   if (ch >= 0x80) {
     intptr_t num_trail_bytes = kTrailBytes[ch];
     bool is_malformed = false;
     for (; i < num_trail_bytes; ++i) {
       if (i < array_len) {
         uint8_t code_unit = utf8_array[i];
         is_malformed |= !IsTrailByte(code_unit);
         ch = (ch << 6) + code_unit;
       } else {
-        *dst = kInvalidCodePoint;
+        *dst = -1;
         return 0;
       }
     }
     ch -= kMagicBits[num_trail_bytes];
     if (!((is_malformed == false) &&
           (i == num_trail_bytes) &&
           !IsOutOfRange(ch) &&
           !IsNonShortestForm(ch, i) &&
           !Utf16::IsSurrogate(ch))) {
-      *dst = kInvalidCodePoint;
+      *dst = -1;
       return 0;
     }
   }
   *dst = ch;
   return i;
 }
 
 
 bool Utf8::DecodeToLatin1(const uint8_t* utf8_array,
                           intptr_t array_len,
@@ skipping 23 matching lines @@
                          intptr_t array_len,
                          uint16_t* dst,
                          intptr_t len) {
   intptr_t i = 0;
   intptr_t j = 0;
   intptr_t num_bytes;
   for (; (i < array_len) && (j < len); i += num_bytes, ++j) {
     int32_t ch;
     bool is_supplementary = IsSupplementarySequenceStart(utf8_array[i]);
     num_bytes = Utf8::Decode(&utf8_array[i], (array_len - i), &ch);
-    if (ch == kInvalidCodePoint) {
+    if (ch == -1) {
       return false;  // invalid input
     }
     if (is_supplementary) {
       Utf16::Encode(ch, &dst[j]);
       j = j + 1;
     } else {
       dst[j] = ch;
     }
   }
   if ((i < array_len) && (j == len)) {
     return false;  // output overflow
   }
   return true;  // success
 }
 
 
 bool Utf8::DecodeToUTF32(const uint8_t* utf8_array,
                          intptr_t array_len,
-                         int32_t* dst,
+                         uint32_t* dst,
                          intptr_t len) {
   intptr_t i = 0;
   intptr_t j = 0;
   intptr_t num_bytes;
   for (; (i < array_len) && (j < len); i += num_bytes, ++j) {
     int32_t ch;
     num_bytes = Utf8::Decode(&utf8_array[i], (array_len - i), &ch);
-    if (ch == kInvalidCodePoint) {
+    if (ch == -1) {
       return false;  // invalid input
     }
     dst[j] = ch;
   }
   if ((i < array_len) && (j == len)) {
     return false;  // output overflow
   }
   return true;  // success
 }
 
 
-int32_t Utf16::CodePointAt(const String& str, int index) {
-  int32_t code = str.CharAt(index);
-  if (!IsLeadSurrogate(code)) return code;
-  if (index + 1 == str.Length()) return code;
-  int32_t trail = str.CharAt(index + 1);
-  if (!IsTrailSurrogate(trail)) return code;
-  return Decode(code, trail);
-}
-
-
-void Utf16::Encode(int32_t codePoint, uint16_t* dst) {
-  ASSERT(codePoint > kMaxBmpCodepoint);
+void Utf16::Encode(int32_t codepoint, uint16_t* dst) {
+  ASSERT(codepoint > kMaxBmpCodepoint);
   ASSERT(dst != NULL);
-  dst[0] = LeadFromCodePoint(codePoint);
-  dst[1] = TrailFromCodePoint(codePoint);
+  dst[0] = (Utf16::kLeadSurrogateOffset + (codepoint >> 10));
+  dst[1] = (0xDC00 + (codepoint & 0x3FF));
 }
 
 }  // namespace dart