Add some support for the code-point code-unit distinction.

runtime/lib/string_base.dart

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 /**
  * [_StringBase] contains common methods used by concrete String
  * implementations, e.g., _OneByteString.
  */
 class _StringBase {
 
   factory _StringBase._uninstantiable() {
     throw new UnsupportedError(
         "_StringBase can't be instaniated");
   }
 
   int get hashCode native "String_getHashCode";
 
   /**
    *  Create the most efficient string representation for specified
    *  [codePoints].
    */
-  static String createFromCharCodes(List<int> charCodes) {
+  static String createFromCharCodes(List<int> codePoints) {
     _ObjectArray objectArray;
-    if (charCodes is _ObjectArray) {
-      objectArray = charCodes;
+    if (codePoints is _ObjectArray) {
+      objectArray = codePoints;
     } else {
-      int len = charCodes.length;
+      int len = codePoints.length;
       objectArray = new _ObjectArray(len);
       for (int i = 0; i < len; i++) {
-        objectArray[i] = charCodes[i];
+        objectArray[i] = codePoints[i];
       }
     }
     return _createFromCodePoints(objectArray);
   }
 
   static String _createFromCodePoints(List<int> codePoints)
       native "StringBase_createFromCodePoints";
 
+  static String createFromCodeUnits(List<int> codeUnits) {
+    _ObjectArray objectArray;
+    if (codeUnits is _ObjectArray) {
+      objectArray = codeUnits;
+    } else {
+      int len = codeUnits.length;
+      objectArray = new _ObjectArray(len);
+      for (int i = 0; i < len; i++) {
+        objectArray[i] = codeUnits[i];
+      }
+    }
+    return _createFromCodeUnits(objectArray);
+  }
+
+  static String _createFromCodeUnits(List<int> codeUnits)
+      native "StringBase_createFromCodeUnits";
+
   String operator [](int index) native "String_charAt";
 
   int charCodeAt(int index) native "String_charCodeAt";
 
+  int codeUnitAt(int index) native "String_codeUnitAt";
+
   int get length native "String_getLength";
 
   bool get isEmpty {
     return this.length === 0;
   }
 
   String concat(String other) native "String_concat";
 
   String toString() {
     return this;
   }
 
   bool operator ==(Object other) {
     if (this === other) {
       return true;
     }
     if ((other is !String) ||
         (this.length != other.length)) {
       // TODO(5413632): Compare hash codes when both are present.
       return false;
     }
     return this.compareTo(other) === 0;
   }
 
   int compareTo(String other) {
     int thisLength = this.length;
     int otherLength = other.length;
     int len = (thisLength < otherLength) ? thisLength : otherLength;
     for (int i = 0; i < len; i++) {
-      int thisCodePoint = this.charCodeAt(i);
-      int otherCodePoint = other.charCodeAt(i);
-      if (thisCodePoint < otherCodePoint) {
+      int thisCodeUnit = this.codeUnitAt(i);
+      int otherCodeUnit = other.codeUnitAt(i);
+      if (thisCodeUnit < otherCodeUnit) {
         return -1;
       }
-      if (thisCodePoint > otherCodePoint) {
+      if (thisCodeUnit > otherCodeUnit) {
         return 1;
       }
     }
     if (thisLength < otherLength) return -1;
     if (thisLength > otherLength) return 1;
     return 0;
   }
 
   bool _substringMatches(int start, String other) {
     if (other.isEmpty) return true;
     if ((start < 0) || (start >= this.length)) {
       return false;
     }
     final int len = other.length;
     if ((start + len) > this.length) {
       return false;
     }
     for (int i = 0; i < len; i++) {
-      if (this.charCodeAt(i + start) != other.charCodeAt(i)) {
+      if (this.codeUnitAt(i + start) != other.codeUnitAt(i)) {
         return false;
       }
     }
     return true;
   }
 
   bool endsWith(String other) {
     return _substringMatches(this.length - other.length, other);
   }
 
@@ skipping 48 matching lines @@
     return _substringUnchecked(startIndex, endIndex);
   }
 
   String _substringUnchecked(int startIndex, int endIndex)
       native "StringBase_substringUnchecked";
 
   String trim() {
     final int len = this.length;
     int first = 0;
     for (; first < len; first++) {
-      if (!_isWhitespace(this.charCodeAt(first))) {
+      // There are no whitespace characters that are outside the BMP so we
+      // can use code units here for efficiency.
+      if (!_isWhitespace(this.codeUnitAt(first))) {
         break;
       }
     }
     if (len == first) {
       // String contains only whitespaces.
       return "";
     }
     int last = len - 1;
     for (; last >= first; last--) {
-      if (!_isWhitespace(this.charCodeAt(last))) {
+      if (!_isWhitespace(this.codeUnitAt(last))) {
         break;
       }
     }
     if ((first == 0) && (last == (len - 1))) {
       // Returns this string if it does not have leading or trailing
       // whitespaces.
       return this;
     } else {
       return _substringUnchecked(first, last + 1);
     }
@@ skipping 100 matching lines @@
       if (startIndex == endIndex && endIndex == previousIndex) {
         ++startIndex;  // empty match, advance and restart
         continue;
       }
       result.add(this.substring(previousIndex, match.start));
       startIndex = previousIndex = endIndex;
     }
     return result;
   }
 
+  // TODO(erikcorry): Fix this to use the new code point iterator when it is
+  // available.
   List<String> splitChars() {
     int len = this.length;
     final result = new List<String>(len);
-    for (int i = 0; i < len; i++) {
-      result[i] = this[i];
+    int i, j;
+    for (i = j = 0; i < len; i++, j++) {
+      int c = charCodeAt(i);
+      // Check for non-basic plane character encoded as a UTF-16 surrogate pair.
+      if (c > 0xffff) {

[floitsch, 2012/11/08 15:28:21]

Can't you use Utf16::IsSurrogate(c)?

[erikcorry, 2012/11/15 13:28:25]

No, that's a C++ function.  I added some named constants to make this less ugly.

+        i++;
+      }
+      result[j] = new String.fromCharCodes([c]);
     }
-    return result;
+    if (i == j) return result;
+    // If we saw some non-basic plane characters, then we have to return a
+    // slightly smaller array than expected (we can't trim the original one
+    // because it is non-extendable).  This rarely happens so this is preferable
+    // to having a separate pass over the string to count the code points.
+    final newResult = new List<String>(j);
+    for (i = 0; i < j; i++) newResult[i] = result[i];
+    return newResult;
   }
 
   List<int> get charCodes {
     int len = this.length;
     final result = new List<int>(len);
+    int i, j;
+    for (i = j = 0; i < len; i++, j++) {
+      int c = this.charCodeAt(i);
+      // Check for non-basic plane character encoded as a UTF-16 surrogate pair.
+      if (c > 0xffff) {
+        i++;
+      }
+      result[j] = c;
+    }
+    if (i == j) return result;
+    // If we saw some non-basic plane characters, then we have to return a
+    // slightly smaller array than expected (we can't trim the original one
+    // because it is non-extendable).  This rarely happens so this is preferable
+    // to having a separate pass over the string to count the code points.
+    final newResult = new List<int>(j);
+    for (i = 0; i < j; i++) newResult[i] = result[i];
+    return newResult;
+  }
+
+  List<int> get codeUnits {
+    int len = this.length;
+    final result = new List<int>(len);
     for (int i = 0; i < len; i++) {
-      result[i] = this.charCodeAt(i);
+      result[i] = this.codeUnitAt(i);
     }
     return result;
   }
 
   String toUpperCase() native "String_toUpperCase";
 
   String toLowerCase() native "String_toLowerCase";
 
   // Implementations of Strings methods follow below.
   static String join(List<String> strings, String separator) {
@@ skipping 33 matching lines @@
       native "Strings_concatAll";
 }
 
 
 class _OneByteString extends _StringBase implements String {
   factory _OneByteString._uninstantiable() {
     throw new UnsupportedError(
         "_OneByteString can only be allocated by the VM");
   }
 
-  // Checks for one-byte whitespaces only.
-  // TODO(srdjan): Investigate if 0x85 (NEL) and 0xA0 (NBSP) are valid
-  // whitespaces for one byte strings.
   bool _isWhitespace(int codePoint) {
     return
-      (codePoint === 32) || // Space.
+      (codePoint == 32) ||  // Space.
+      (codePoint == 0xa0) ||  // No-break space.
       ((9 <= codePoint) && (codePoint <= 13)); // CR, LF, TAB, etc.
   }
 
 }
 
 
 class _TwoByteString extends _StringBase implements String {
   factory _TwoByteString._uninstantiable() {
     throw new UnsupportedError(
         "_TwoByteString can only be allocated by the VM");
   }
 
-  // Checks for one-byte whitespaces only.
-  // TODO(srdjan): Investigate if 0x85 (NEL) and 0xA0 (NBSP) are valid
-  // whitespaces. Add checking for multi-byte whitespace codepoints.
+  // Works for both code points and code units since all spaces are in the BMP.
   bool _isWhitespace(int codePoint) {
     return
-      (codePoint === 32) || // Space.
-      ((9 <= codePoint) && (codePoint <= 13)); // CR, LF, TAB, etc.
+      (codePoint == 32) ||  // Space.
+      (codePoint == 0xa0) ||  // No-break space.
+      ((9 <= codePoint) && (codePoint <= 13)) ||  // CR, LF, TAB, etc.
+      (codePoint >= 0x1680 &&  // Optimization.
+       (codePoint == 0x1680 ||  // Ogham space mark.
+        codePoint == 0x180e ||  // Mongolian vowel separator.
+        (codePoint >= 0x2000 && codePoint <= 0x200a) ||  // Wide/narrow spaces.
+        codePoint == 0x202f ||  // Narrow no-break space.
+        codePoint == 0x205f ||  // Medium mathematical space.
+        codePoint == 0x3000));  // Ideographic space.
   }
 }
 
 
+// TODO(erikcorry): This is going away.
 class _FourByteString extends _StringBase implements String {
   factory _FourByteString._uninstantiable() {
     throw new UnsupportedError(
         "_FourByteString can only be allocated by the VM");
   }
 
   // Checks for one-byte whitespaces only.
   // TODO(srdjan): Investigate if 0x85 (NEL) and 0xA0 (NBSP) are valid
   // whitespaces. Add checking for multi-byte whitespace codepoints.
   bool _isWhitespace(int codePoint) {
     return
       (codePoint === 32) || // Space.
       ((9 <= codePoint) && (codePoint <= 13)); // CR, LF, TAB, etc.
   }
 }
 
 
 class _ExternalOneByteString extends _StringBase implements String {
   factory _ExternalOneByteString._uninstantiable() {
     throw new UnsupportedError(
         "_ExternalOneByteString can only be allocated by the VM");
   }
 
-  // Checks for one-byte whitespaces only.
-  // TODO(srdjan): Investigate if 0x85 (NEL) and 0xA0 (NBSP) are valid
-  // whitespaces for one byte strings.
   bool _isWhitespace(int codePoint) {
     return
-      (codePoint === 32) || // Space.
+      (codePoint == 32) ||  // Space.
+      (codePoint == 0xa0) ||  // No-break space.
       ((9 <= codePoint) && (codePoint <= 13)); // CR, LF, TAB, etc.
   }
 }
 
 
 class _ExternalTwoByteString extends _StringBase implements String {
   factory ExternalTwoByteString._uninstantiable() {
     throw new UnsupportedError(
         "_ExternalTwoByteString can only be allocated by the VM");
   }
 
-  // Checks for one-byte whitespaces only.
-  // TODO(srdjan): Investigate if 0x85 (NEL) and 0xA0 (NBSP) are valid
-  // whitespaces. Add checking for multi-byte whitespace codepoints.
+  // Works for both code points and code units since all spaces are in the BMP.
   bool _isWhitespace(int codePoint) {
     return
-      (codePoint === 32) || // Space.
-      ((9 <= codePoint) && (codePoint <= 13)); // CR, LF, TAB, etc.
+      (codePoint == 32) ||  // Space.
+      (codePoint == 0xa0) ||  // No-break space.
+      ((9 <= codePoint) && (codePoint <= 13)) ||  // CR, LF, TAB, etc.
+      (codePoint >= 0x1680 &&  // Optimization.
+       (codePoint == 0x1680 ||  // Ogham space mark.
+        codePoint == 0x180e ||  // Mongolian vowel separator.
+        (codePoint >= 0x2000 && codePoint <= 0x200a) ||  // Wide/narrow spaces.
+        codePoint == 0x202f ||  // Narrow no-break space.
+        codePoint == 0x205f ||  // Medium mathematical space.
+        codePoint == 0x3000));  // Ideographic space.
   }
 }
 
 
+// TODO(erikcorry): This is going away.
 class _ExternalFourByteString extends _StringBase implements String {
   factory _ExternalFourByteString._uninstantiable() {
     throw new UnsupportedError(
         "ExternalFourByteString can only be allocated by the VM");
   }
 
   // Checks for one-byte whitespaces only.
   // TODO(srdjan): Investigate if 0x85 (NEL) and 0xA0 (NBSP) are valid
   // whitespaces. Add checking for multi-byte whitespace codepoints.
   bool _isWhitespace(int codePoint) {
@@ skipping 25 matching lines @@
     for (int g in groups) {
       result.add(group(g));
     }
     return result;
   }
 
   final int start;
   final String str;
   final String pattern;
 }