Add some support for the code-point code-unit distinction.

runtime/vm/object.h

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #ifndef VM_OBJECT_H_
 #define VM_OBJECT_H_
 
 #include "include/dart_api.h"
 #include "platform/assert.h"
 #include "platform/utils.h"
 #include "vm/bitmap.h"
 #include "vm/dart.h"
 #include "vm/globals.h"
 #include "vm/handles.h"
 #include "vm/heap.h"
 #include "vm/isolate.h"
 #include "vm/os.h"
 #include "vm/raw_object.h"
 #include "vm/scanner.h"
+#include "vm/unicode.h"
 
 namespace dart {
 
 // Forward declarations.
 #define DEFINE_FORWARD_DECLARATION(clazz)                                      \
   class clazz;
 CLASS_LIST(DEFINE_FORWARD_DECLARATION)
 #undef DEFINE_FORWARD_DECLARATION
 class Api;
 class Assembler;
@@ skipping 3647 matching lines @@
   intptr_t Length() const { return Smi::Value(raw_ptr()->length_); }
   static intptr_t length_offset() { return OFFSET_OF(RawString, length_); }
 
   virtual intptr_t Hash() const;
   static intptr_t hash_offset() { return OFFSET_OF(RawString, hash_); }
   static intptr_t Hash(const String& str, intptr_t begin_index, intptr_t len);
   static intptr_t Hash(const uint8_t* characters, intptr_t len);
   static intptr_t Hash(const uint16_t* characters, intptr_t len);
   static intptr_t Hash(const uint32_t* characters, intptr_t len);
 
-  int32_t CharAt(intptr_t index) const;
+  uint32_t CharAt(intptr_t index) const;
+  uint32_t CodeUnitAt(intptr_t index) const;
 
   intptr_t CharSize() const;
 
   inline bool Equals(const String& str) const;
   inline bool Equals(const String& str,
                      intptr_t begin_index,  // begin index on 'str'.
                      intptr_t len) const;  // len on 'str'.
   bool Equals(const char* str) const;
   bool Equals(const uint8_t* characters, intptr_t len) const;
   bool Equals(const uint16_t* characters, intptr_t len) const;
@@ skipping 145 matching lines @@
   friend class Symbols;
   friend class OneByteString;
   friend class TwoByteString;
   friend class ExternalOneByteString;
   friend class ExternalTwoByteString;
 };
 
 
 class OneByteString : public AllStatic {
  public:
-  static int32_t CharAt(const String& str, intptr_t index) {
+  static uint32_t CharAt(const String& str, intptr_t index) {
     return *CharAddr(str, index);
   }
 
+  static uint32_t CodeUnitAt(const String& str, intptr_t index) {
+    return *CharAddr(str, index);
+  }
+
   static RawOneByteString* EscapeSpecialCharacters(const String& str,
                                                    bool raw_str);
 
   static bool EqualsIgnoringPrivateKey(const String& str1,
                                        const String& str2);
 
   // We use the same maximum elements for all strings.
   static const intptr_t kBytesPerElement = 1;
   static const intptr_t kMaxElements = String::kMaxElements;
 
@@ skipping 33 matching lines @@
 
   static RawOneByteString* Concat(const String& str1,
                                   const String& str2,
                                   Heap::Space space);
   static RawOneByteString* ConcatAll(const Array& strings,
                                      intptr_t len,
                                      Heap::Space space);
 
   static RawOneByteString* Transform(int32_t (*mapping)(int32_t ch),
                                      const String& str,
+                                     int out_length,
                                      Heap::Space space);
 
   static const ClassId kClassId = kOneByteStringCid;
 
   static RawOneByteString* null() {
     return reinterpret_cast<RawOneByteString*>(Object::null());
   }
 
  private:
   static RawOneByteString* raw(const String& str) {
@@ skipping 17 matching lines @@
                                     Snapshot::Kind kind);
 
   friend class Class;
   friend class String;
   friend class SnapshotReader;
 };
 
 
 class TwoByteString : public AllStatic {
  public:
-  static int32_t CharAt(const String& str, intptr_t index) {
+  static uint32_t CharAt(const String& str, intptr_t index) {
+    int32_t unit = *CharAddr(str, index);
+    // For non-surrogate values or incorrect trailing surrogates we just return
+    // the value.
+    if (!Utf16::IsLeadSurrogate(unit)) return unit;
+
+    // If the string ends with a lead surrogate we just return that.
+    if (index + 1 >= str.Length()) return unit;
+
+    return Utf16::CodePointFromCodeUnits(unit, *CharAddr(str, index + 1));
+  }
+
+  static uint32_t CodeUnitAt(const String& str, intptr_t index) {
     return *CharAddr(str, index);
   }

[siva, 2012/11/08 19:06:47]

I don't mind changing the  name 'CharAt' to 'CodeUnitAt" but
I don't think your new 'CharAt' functionality belongs
in the VM code.

Iteration of characters in a string should be done in an
Iterator implementation in the library.

[erikcorry, 2012/11/08 22:09:34]

Do you want to move the C++ support for toUpperCase and toLowerCase into dart
too?  They make use of charAt.

[siva, 2012/11/09 02:40:42]

We could fix String::Transform/TwoByteString::Transform
to locally compute the unicode values as it iterates over
the string.

On 2012/11/08 22:09:34, erikcorry wrote:

[erikcorry, 2012/11/15 13:28:25]

I have fixed the UTF8 and the Transform (toUpper/Lower) routines to use a helper
from unicode.h and removed this from the String classes.

 
   static RawTwoByteString* EscapeSpecialCharacters(const String& str,
                                                    bool raw_str);
 
   // We use the same maximum elements for all strings.
   static const intptr_t kBytesPerElement = 2;
   static const intptr_t kMaxElements = String::kMaxElements;
 
   static intptr_t data_offset() { return OFFSET_OF(RawTwoByteString, data_); }
 
@@ skipping 23 matching lines @@
 
   static RawTwoByteString* Concat(const String& str1,
                                   const String& str2,
                                   Heap::Space space);
   static RawTwoByteString* ConcatAll(const Array& strings,
                                      intptr_t len,
                                      Heap::Space space);
 
   static RawTwoByteString* Transform(int32_t (*mapping)(int32_t ch),
                                      const String& str,
+                                     int out_length,
                                      Heap::Space space);
 
   static RawTwoByteString* null() {
     return reinterpret_cast<RawTwoByteString*>(Object::null());
   }
 
 
   static const ClassId kClassId = kTwoByteStringCid;
 
  private:
@@ skipping 18 matching lines @@
                                     Snapshot::Kind kind);
 
   friend class Class;
   friend class String;
   friend class SnapshotReader;
 };
 
 
 class ExternalOneByteString : public AllStatic {
  public:
-  static int32_t CharAt(const String& str, intptr_t index) {
+  static uint32_t CharAt(const String& str, intptr_t index) {
     return *CharAddr(str, index);
   }
 
+  static uint32_t CodeUnitAt(const String& str, intptr_t index) {
+    return *CharAddr(str, index);
+  }
+
   static void* GetPeer(const String& str) {
     return raw_ptr(str)->external_data_->peer();
   }
 
   // We use the same maximum elements for all strings.
   static const intptr_t kBytesPerElement = 1;
   static const intptr_t kMaxElements = String::kMaxElements;
 
   static intptr_t InstanceSize() {
     return String::RoundedAllocationSize(sizeof(RawExternalOneByteString));
@@ skipping 42 matching lines @@
                                             Snapshot::Kind kind);
 
   friend class Class;
   friend class String;
   friend class SnapshotReader;
 };
 
 
 class ExternalTwoByteString : public AllStatic {
  public:
-  static int32_t CharAt(const String& str, intptr_t index) {
+  static uint32_t CharAt(const String& str, intptr_t index) {
+    int32_t unit = *CharAddr(str, index);

[floitsch, 2012/11/08 15:28:21]

uint32_t unit =

[erikcorry, 2012/11/15 13:28:25]

This code is gone.

+    // For non-surrogate values or incorrect trailing surrogates we just return
+    // the value.
+    if (!Utf16::IsLeadSurrogate(unit)) return unit;
+
+    // If the string ends with a lead surrogate we just return that.
+    if (index + 1 >= str.Length()) return unit;
+
+    return Utf16::CodePointFromCodeUnits(unit, *CharAddr(str, index + 1));
+  }
+
+  static uint32_t CodeUnitAt(const String& str, intptr_t index) {
     return *CharAddr(str, index);
   }
 
   static void* GetPeer(const String& str) {
     return raw_ptr(str)->external_data_->peer();
   }
 
   // We use the same maximum elements for all strings.
   static const intptr_t kBytesPerElement = 2;
   static const intptr_t kMaxElements = String::kMaxElements;
@@ skipping 1743 matching lines @@
                     intptr_t begin_index,
                     intptr_t len) const {
   ASSERT(begin_index >= 0);
   ASSERT((begin_index == 0) || (begin_index < str.Length()));
   ASSERT(len >= 0);
   ASSERT(len <= str.Length());
   if (len != this->Length()) {
     return false;  // Lengths don't match.
   }
   for (intptr_t i = 0; i < len; i++) {
-    if (this->CharAt(i) != str.CharAt(begin_index + i)) {
+    if (this->CodeUnitAt(i) != str.CodeUnitAt(begin_index + i)) {
       return false;
     }
   }
   return true;
 }
 
 }  // namespace dart
 
 #endif  // VM_OBJECT_H_