Index: runtime/vm/unicode.cc |
diff --git a/runtime/vm/unicode.cc b/runtime/vm/unicode.cc |
index e2e741d848339162fd9fa7f89924ed8363e1469e..b6303c5be82ad42bdb3175e1e5a4c8cc07b8e737 100644 |
--- a/runtime/vm/unicode.cc |
+++ b/runtime/vm/unicode.cc |
@@ -48,8 +48,8 @@ static const uint32_t kOverlongMinimum[7] = { |
0x80, |
0x800, |
0x10000, |
- 0xFFFFFFFF, |
- 0xFFFFFFFF |
+ 0xFFFFFFFF, // We never allow 5 byte sequences. |
+ 0xFFFFFFFF // We never allow 6 byte sequences. |
}; |
@@ -65,41 +65,30 @@ static bool IsLatin1SequenceStart(uint8_t code_unit) { |
static bool IsSmpSequenceStart(uint8_t code_unit) { |
- // Check is codepoint is >= U+10000. |
+ // Check the UTF-8 code unit to determine if it is a sequence start for a |
+ // code point >= U+10000. |
return (code_unit >= 0xF0); |
} |
-// Returns true if the code point is a high- or low-surrogate. |
-static bool IsSurrogate(uint32_t code_point) { |
- return (code_point & 0xfffff800) == 0xd800; |
-} |
- |
- |
// Returns true if the code point value is above Plane 17. |
-static bool IsOutOfRange(uint32_t code_point) { |
- return (code_point > 0x10FFFF); |
+static bool IsOutOfRange(int32_t code_point) { |
+ return (code_point > Utf16::kMaxCodePoint); |
} |
// Returns true if the byte sequence is ill-formed. |
-static bool IsNonShortestForm(uint32_t code_point, size_t num_bytes) { |
- return code_point < kOverlongMinimum[num_bytes]; |
-} |
- |
- |
-void Utf8::ConvertUTF32ToUTF16(int32_t codepoint, uint16_t* dst) { |
- ASSERT(codepoint > kMaxBmpCodepoint); |
- ASSERT(dst != NULL); |
- dst[0] = (Utf8::kLeadOffset + (codepoint >> 10)); |
- dst[1] = (0xDC00 + (codepoint & 0x3FF)); |
+static bool IsNonShortestForm(int32_t code_point, size_t num_bytes) { |
+ return static_cast<uint32_t>(code_point) < kOverlongMinimum[num_bytes]; |
} |
-// Returns a count of the number of UTF-8 trail bytes. |
-intptr_t Utf8::CodePointCount(const uint8_t* utf8_array, |
- intptr_t array_len, |
- Type* type) { |
+// Returns a count of the number of UTF-16 code units represented by this UTF-8 |
+// array. Type is kASCII for 7-bit-only. If there are surrogate pairs then |
+// the type is kSMP. Otherwise it is kBMP. |
+intptr_t Utf8::CodeUnitCount(const uint8_t* utf8_array, |
+ intptr_t array_len, |
+ Type* type) { |
intptr_t len = 0; |
Type char_type = kLatin1; |
for (intptr_t i = 0; i < array_len; i++) { |
@@ -110,7 +99,7 @@ intptr_t Utf8::CodePointCount(const uint8_t* utf8_array, |
if (!IsLatin1SequenceStart(code_unit)) { // > U+00FF |
if (IsSmpSequenceStart(code_unit)) { // >= U+10000 |
char_type = kSMP; |
- ++len; |
+ ++len; // Surrogate pair in the UTF-16 encoding. |
} else if (char_type == kLatin1) { |
char_type = kBMP; |
} |
@@ -121,7 +110,7 @@ intptr_t Utf8::CodePointCount(const uint8_t* utf8_array, |
} |
-// Returns true if str is a valid NUL-terminated UTF-8 string. |
+// Returns true if str is a valid UTF-8 string. |
bool Utf8::IsValid(const uint8_t* utf8_array, intptr_t array_len) { |
intptr_t i = 0; |
while (i < array_len) { |
@@ -144,7 +133,7 @@ bool Utf8::IsValid(const uint8_t* utf8_array, intptr_t array_len) { |
(j == num_trail_bytes) && |
!IsOutOfRange(ch) && |
!IsNonShortestForm(ch, j) && |
- !IsSurrogate(ch))) { |
+ !Utf16::IsSurrogate(ch))) { |
return false; |
} |
} |
@@ -170,8 +159,9 @@ intptr_t Utf8::Length(int32_t ch) { |
intptr_t Utf8::Length(const String& str) { |
intptr_t length = 0; |
for (intptr_t i = 0; i < str.Length(); ++i) { |
- int32_t ch = str.CharAt(i); |
+ int32_t ch = Utf16::CodePointAt(str, i); |
length += Utf8::Length(ch); |
+ if (ch >= 0x10000) i++; // Surrogate pair in input |
} |
return length; |
} |
@@ -206,13 +196,14 @@ intptr_t Utf8::Encode(int32_t ch, char* dst) { |
intptr_t Utf8::Encode(const String& src, char* dst, intptr_t len) { |
intptr_t pos = 0; |
for (intptr_t i = 0; i < src.Length(); ++i) { |
- intptr_t ch = src.CharAt(i); |
+ intptr_t ch = Utf16::CodePointAt(src, i); |
intptr_t num_bytes = Utf8::Length(ch); |
if (pos + num_bytes > len) { |
break; |
} |
Utf8::Encode(ch, &dst[pos]); |
pos += num_bytes; |
+ if (num_bytes > 3) i++; // Surrogate pair in input. |
} |
return pos; |
} |
@@ -221,7 +212,7 @@ intptr_t Utf8::Encode(const String& src, char* dst, intptr_t len) { |
intptr_t Utf8::Decode(const uint8_t* utf8_array, |
intptr_t array_len, |
int32_t* dst) { |
- uint32_t ch = utf8_array[0] & 0xFF; |
+ int32_t ch = utf8_array[0] & 0xFF; |
intptr_t i = 1; |
if (ch >= 0x80) { |
int32_t num_trail_bytes = kTrailBytes[ch]; |
@@ -232,7 +223,7 @@ intptr_t Utf8::Decode(const uint8_t* utf8_array, |
is_malformed |= !IsTrailByte(code_unit); |
ch = (ch << 6) + code_unit; |
} else { |
- *dst = -1; |
+ *dst = kInvalidCodePoint; |
return 0; |
} |
} |
@@ -241,8 +232,8 @@ intptr_t Utf8::Decode(const uint8_t* utf8_array, |
(i == num_trail_bytes) && |
!IsOutOfRange(ch) && |
!IsNonShortestForm(ch, i) && |
- !IsSurrogate(ch))) { |
- *dst = -1; |
+ !Utf16::IsSurrogate(ch))) { |
+ *dst = kInvalidCodePoint; |
return 0; |
} |
} |
@@ -286,12 +277,13 @@ bool Utf8::DecodeToUTF16(const uint8_t* utf8_array, |
int32_t ch; |
bool is_smp = IsSmpSequenceStart(utf8_array[i]); |
num_bytes = Utf8::Decode(&utf8_array[i], (array_len - i), &ch); |
- if (ch == -1) { |
+ if (ch == kInvalidCodePoint) { |
return false; // invalid input |
} |
if (is_smp) { |
- ConvertUTF32ToUTF16(ch, &(dst[j])); |
- j = j + 1; |
+ dst[j] = Utf16::LeadFromCodePoint(ch); |
+ dst[j + 1] = Utf16::TrailFromCodePoint(ch); |
+ ++j; |
} else { |
dst[j] = ch; |
} |
@@ -305,7 +297,7 @@ bool Utf8::DecodeToUTF16(const uint8_t* utf8_array, |
bool Utf8::DecodeToUTF32(const uint8_t* utf8_array, |
intptr_t array_len, |
- uint32_t* dst, |
+ int32_t* dst, |
intptr_t len) { |
intptr_t i = 0; |
intptr_t j = 0; |
@@ -313,7 +305,7 @@ bool Utf8::DecodeToUTF32(const uint8_t* utf8_array, |
for (; (i < array_len) && (j < len); i += num_bytes, ++j) { |
int32_t ch; |
num_bytes = Utf8::Decode(&utf8_array[i], (array_len - i), &ch); |
- if (ch == -1) { |
+ if (ch == kInvalidCodePoint) { |
return false; // invalid input |
} |
dst[j] = ch; |
@@ -324,4 +316,14 @@ bool Utf8::DecodeToUTF32(const uint8_t* utf8_array, |
return true; // success |
} |
+ |
+int32_t Utf16::CodePointAt(const String& str, int index) { |
+ int32_t code = str.CharAt(index); |
+ if (!IsLeadSurrogate(code)) return code; |
+ if (index + 1 == str.Length()) return code; |
+ int32_t trail = str.CharAt(index + 1); |
+ if (!IsTrailSurrogate(trail)) return code; |
+ return CodePointFromCodeUnits(code, trail); |
+} |
+ |
} // namespace dart |