Updated unicode library.

test/cctest/test-regexp.cc

 // Copyright 2008 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 1381 matching lines @@
     CHECK_EQ(canonicalize(lower), canonicalize(upper));
     unibrow::uchar uncanon[unibrow::Ecma262UnCanonicalize::kMaxWidth];
     int length = un_canonicalize.get(lower, '\0', uncanon);
     CHECK_EQ(2, length);
     CHECK_EQ(upper, uncanon[0]);
     CHECK_EQ(lower, uncanon[1]);
   }
   for (uc32 c = 128; c < (1 << 21); c++)
     CHECK_GE(canonicalize(c), 128);
   unibrow::Mapping<unibrow::ToUppercase> to_upper;
-  for (uc32 c = 0; c < (1 << 21); c++) {
+  // Canonicalization is only defined for the Basic Multilingual Plane.
+  for (uc32 c = 0; c < (1 << 16); c++) {
     unibrow::uchar upper[unibrow::ToUppercase::kMaxWidth];
     int length = to_upper.get(c, '\0', upper);
     if (length == 0) {
       length = 1;
       upper[0] = c;
     }
     uc32 u = upper[0];
     if (length > 1 || (c >= 128 && u < 128))
       u = c;
     CHECK_EQ(u, canonicalize(c));
   }
 }
 
 
-static uc32 CanonRange(uc32 c) {
+static uc32 CanonRangeEnd(uc32 c) {
   unibrow::uchar canon[unibrow::CanonicalizationRange::kMaxWidth];
   int count = unibrow::CanonicalizationRange::Convert(c, '\0', canon, NULL);
   if (count == 0) {
     return c;
   } else {
     CHECK_EQ(1, count);
     return canon[0];
   }
 }
 
 
 TEST(RangeCanonicalization) {
-  CHECK_NE(CanonRange(0) & CharacterRange::kStartMarker, 0);
   // Check that we arrive at the same result when using the basic
   // range canonicalization primitives as when using immediate
   // canonicalization.
   unibrow::Mapping<unibrow::Ecma262UnCanonicalize> un_canonicalize;
-  for (int i = 0; i < CharacterRange::kRangeCanonicalizeMax; i++) {
-    int range = CanonRange(i);
-    int indirect_length = 0;
-    unibrow::uchar indirect[unibrow::Ecma262UnCanonicalize::kMaxWidth];
-    if ((range & CharacterRange::kStartMarker) == 0) {
-      indirect_length = un_canonicalize.get(i - range, '\0', indirect);
-      for (int i = 0; i < indirect_length; i++)
-        indirect[i] += range;
-    } else {
-      indirect_length = un_canonicalize.get(i, '\0', indirect);
-    }
-    unibrow::uchar direct[unibrow::Ecma262UnCanonicalize::kMaxWidth];
-    int direct_length = un_canonicalize.get(i, '\0', direct);
-    CHECK_EQ(direct_length, indirect_length);
-  }
-  // Check that we arrive at the same results when skipping over
-  // canonicalization ranges.
-  int next_block = 0;
-  while (next_block < CharacterRange::kRangeCanonicalizeMax) {
-    uc32 start = CanonRange(next_block);
-    CHECK_NE((start & CharacterRange::kStartMarker), 0);
-    unsigned dist = start & CharacterRange::kPayloadMask;
-    unibrow::uchar first[unibrow::Ecma262UnCanonicalize::kMaxWidth];
-    int first_length = un_canonicalize.get(next_block, '\0', first);
-    for (unsigned i = 1; i < dist; i++) {
-      CHECK_EQ(i, CanonRange(next_block + i));
-      unibrow::uchar succ[unibrow::Ecma262UnCanonicalize::kMaxWidth];
-      int succ_length = un_canonicalize.get(next_block + i, '\0', succ);
-      CHECK_EQ(first_length, succ_length);
-      for (int j = 0; j < succ_length; j++) {
-        int calc = first[j] + i;
-        int found = succ[j];
-        CHECK_EQ(calc, found);
+  int block_start = 0;
+  while (block_start <= 0xFFFF) {
+    uc32 block_end = CanonRangeEnd(block_start);
+    unsigned block_length = block_end - block_start + 1;
+    if (block_length > 1) {
+      unibrow::uchar first[unibrow::Ecma262UnCanonicalize::kMaxWidth];
+      int first_length = un_canonicalize.get(block_start, '\0', first);
+      for (unsigned i = 1; i < block_length; i++) {
+        unibrow::uchar succ[unibrow::Ecma262UnCanonicalize::kMaxWidth];
+        int succ_length = un_canonicalize.get(block_start + i, '\0', succ);
+        CHECK_EQ(first_length, succ_length);
+        for (int j = 0; j < succ_length; j++) {
+          int calc = first[j] + i;
+          int found = succ[j];
+          CHECK_EQ(calc, found);
+        }
       }
     }
-    next_block = next_block + dist;
+    block_start = block_start + block_length;
   }
 }
 
 
 TEST(UncanonicalizeEquivalence) {
   unibrow::Mapping<unibrow::Ecma262UnCanonicalize> un_canonicalize;
   unibrow::uchar chars[unibrow::Ecma262UnCanonicalize::kMaxWidth];
   for (int i = 0; i < (1 << 16); i++) {
     int length = un_canonicalize.get(i, '\0', chars);
     for (int j = 0; j < length; j++) {
@@ skipping 311 matching lines @@
     bool in_second = CharacterInSet(&l2, i);
     CHECK((in_first || in_second) == CharacterInSet(&all, i));
   }
 }
 
 
 TEST(Graph) {
   V8::Initialize(NULL);
   Execute("\\b\\w+\\b", false, true, true);
 }