Fix bug in Utf8::CodePointCount which was causing some strings with latin1

vm/unicode.h

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #ifndef VM_UNICODE_H_
 #define VM_UNICODE_H_
 
 #include "vm/allocation.h"
 #include "vm/globals.h"
 
@@ skipping 25 matching lines @@
 
 
 class Utf8 : AllStatic {
  public:
   enum Type {
     kLatin1 = 0,  // Latin-1 code point [U+0000, U+00FF].
     kBMP,  // Basic Multilingual Plane code point [U+0000, U+FFFF].
     kSupplementary,  // Supplementary code point [U+010000, U+10FFFF].
   };
 
-  static intptr_t CodePointCount(const uint8_t* utf8_array,
-                                 intptr_t array_len,
-                                 Type* type);
+  // Returns a count of the number of UTF-16 code units needed to represent the

[cshapiro, 2012/11/30 21:32:26]

This is not strictly true, right?

This returns the minimal number of code units as well as the most restricted
coding form.

[siva, 2012/11/30 21:47:19]

Changed the comment to:
Returns the most restricted coding form in which the sequence of utf8 characters
in 'utf8_array' can be represented in and the number of code units needed in
that form.

On 2012/11/30 21:32:26, cshapiro wrote:

+  // sequence of utf8 characters in 'utf8_array'.
+  static intptr_t CodeUnitCount(const uint8_t* utf8_array,
+                                intptr_t array_len,
+                                Type* type);
 
   // Returns true if 'utf8_array' is a valid UTF-8 string.
   static bool IsValid(const uint8_t* utf8_array, intptr_t array_len);
 
   static intptr_t Length(int32_t ch);
   static intptr_t Length(const String& str);
 
   static intptr_t Encode(int32_t ch, char* dst);
   static intptr_t Encode(const String& src, char* dst, intptr_t len);
 
@@ skipping 16 matching lines @@
   static bool DecodeCStringToUTF32(const char* str,
                                    int32_t* dst,
                                    intptr_t len);
 
  private:
   static const int32_t kMaxOneByteChar   = 0x7F;
   static const int32_t kMaxTwoByteChar   = 0x7FF;
   static const int32_t kMaxThreeByteChar = 0xFFFF;
   static const int32_t kMaxFourByteChar  = Utf::kMaxCodePoint;
 
-  static bool IsTrailByte(uint8_t code_unit) {
-    return (code_unit & 0xc0) == 0x80;
+  static bool IsTrailByte(uint8_t utf8_byte) {

[cshapiro, 2012/11/30 21:32:26]

the utf-8 spec removed all mention of "byte" and replaced it with code unit.

it should be okay to keep the old name since this is in the utf-8 class now.

[siva, 2012/11/30 21:47:19]

Done.

+    return (utf8_byte & 0xC0) == 0x80;
   }
 
   static bool IsNonShortestForm(uint32_t code_point, size_t num_code_units) {
     return code_point < kOverlongMinimum[num_code_units];
   }
 
-  static bool IsLatin1SequenceStart(uint8_t code_unit) {
-    // Check is codepoint is <= U+00FF
-    return (code_unit <= Utf8::kMaxOneByteChar);
+  static bool IsLatin1SequenceStart(uint8_t utf8_byte) {
+    // Check if utf8 sequence is start of a codepoint <= U+00FF
+    return (utf8_byte <= 0xC3);
   }
 
-  static bool IsSupplementarySequenceStart(uint8_t code_unit) {
-    // Check is codepoint is >= U+10000.
-    return (code_unit >= 0xF0);
+  static bool IsSupplementarySequenceStart(uint8_t utf8_byte) {
+    // Check if utf8 sequence is start of a codepoint >= U+10000.
+    return (utf8_byte >= 0xF0);
   }
 
   static const int8_t kTrailBytes[];
   static const uint32_t kMagicBits[];
   static const uint32_t kOverlongMinimum[];
 };
 
 
 class Utf16 : AllStatic {
  public:
@@ skipping 108 matching lines @@
   // Data for small code points with one mapping
   static const int16_t stage2_[];
 
   // Data for large code points or code points with both mappings.
   static const int32_t stage2_exception_[][2];
 };
 
 }  // namespace dart
 
 #endif  // VM_UNICODE_H_