Update ICU to 54.1 step 1

source/test/intltest/alphaindextst.cpp

 /********************************************************************
  * COPYRIGHT:
- * Copyright (c) 2012-2013, International Business Machines Corporation
+ * Copyright (c) 2012-2014, International Business Machines Corporation
  * and others. All Rights Reserved.
  ********************************************************************/
 //
-//   file:  alphaindex.cpp
+//   file:  alphaindextst.cpp
 //          Alphabetic Index Tests.
 //
 //   Note: please... no character literals cast to UChars.. use (UChar)0xZZZZ
 
 #include <stdio.h>  // for sprintf
 
 #include "intltest.h"
 #include "alphaindextst.h"
+#include "cmemory.h"
 
 #include "unicode/alphaindex.h"
 #include "unicode/coll.h"
 #include "unicode/localpointer.h"
 #include "unicode/tblcoll.h"
 #include "unicode/uniset.h"
 
 #if !UCONFIG_NO_COLLATION && !UCONFIG_NO_NORMALIZATION
 
 // #include <string>
 // #include <iostream>
 
-#define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0]))
-
 namespace {
 
 UnicodeString joinLabelsAndAppend(AlphabeticIndex::ImmutableIndex &index, UnicodeString &dest) {
     int32_t oldLength = dest.length();
     const AlphabeticIndex::Bucket *bucket;
     for (int32_t i = 0; (bucket = index.getBucket(i)) != NULL; ++i) {
         if (dest.length() > oldLength) {
             dest.append((UChar)0x3A);  // ':'
         }
         dest.append(bucket->getLabel());
@@ skipping 16 matching lines @@
     TESTCASE_AUTO(APITest);
     TESTCASE_AUTO(ManyLocalesTest);
     TESTCASE_AUTO(HackPinyinTest);
     TESTCASE_AUTO(TestBug9009);
     TESTCASE_AUTO(TestIndexCharactersList);
     TESTCASE_AUTO(TestHaniFirst);
     TESTCASE_AUTO(TestPinyinFirst);
     TESTCASE_AUTO(TestSchSt);
     TESTCASE_AUTO(TestNoLabels);
     TESTCASE_AUTO(TestChineseZhuyin);
+    TESTCASE_AUTO(TestJapaneseKanji);
+    TESTCASE_AUTO(TestChineseUnihan);
     TESTCASE_AUTO_END;
 }
 
 #define TEST_CHECK_STATUS {if (U_FAILURE(status)) {dataerrln("%s:%d: Test failure.  status=%s", \
                                                               __FILE__, __LINE__, u_errorName(status)); return;}}
 
 #define TEST_ASSERT(expr) {if ((expr)==FALSE) {errln("%s:%d: Test failure \n", __FILE__, __LINE__);};}
 
 //
 //  APITest.   Invoke every function at least once, and check that it does something.
@@ skipping 10 matching lines @@
     TEST_CHECK_STATUS;
     lc = index->getBucketCount(status);
     TEST_CHECK_STATUS;
     TEST_ASSERT(28 == lc);    // 26 letters plus two under/overflow labels.
     //printf("getBucketCount() == %d\n", lc);
     delete index;
 
     // Constructor from a Collator
     //
     status = U_ZERO_ERROR;
-    RuleBasedCollator *coll = dynamic_cast<RuleBasedCollator *>(Collator::createInstance(Locale::getChinese(), status));
+    RuleBasedCollator *coll = dynamic_cast<RuleBasedCollator *>(
+        Collator::createInstance(Locale::getGerman(), status));
     TEST_CHECK_STATUS;
     TEST_ASSERT(coll != NULL);
     index = new AlphabeticIndex(coll, status);
     TEST_CHECK_STATUS;
     TEST_ASSERT(coll == &index->getCollator());
     assertEquals("only the underflow label in an index built from a collator",
                  1, index->getBucketCount(status));
     TEST_CHECK_STATUS;
     delete index;
     
@@ skipping 200 matching lines @@
     delete index;
     index = new AlphabeticIndex(Locale::createFromName("ru"), status);
     TEST_CHECK_STATUS;
     assertEquals("Russian index.getBucketCount()", 32, index->getBucketCount(status));
     // Latin-script names should go into the underflow label (0)
     // if the Russian collation does not use script reordering,
     // but into the overflow label (getBucketCount()-1)
     // if Russian sorts Cyrillic first.
     int32_t reorderCodes[20];
     int32_t expectedLatinIndex = 0;
-    if (index->getCollator().getReorderCodes(reorderCodes, LENGTHOF(reorderCodes), status) > 0) {
+    if (index->getCollator().getReorderCodes(reorderCodes, UPRV_LENGTHOF(reorderCodes), status) > 0) {
         expectedLatinIndex = index->getBucketCount(status) - 1;
     }
     n = index->getBucketIndex(adam, status);
     TEST_CHECK_STATUS;
     assertEquals("Russian index.getBucketIndex(adam)", expectedLatinIndex, n);
     n = index->getBucketIndex(baker, status);
     assertEquals("Russian index.getBucketIndex(baker)", expectedLatinIndex, n);
     n = index->getBucketIndex(Cyrillic, status);
     assertEquals("Russian index.getBucketIndex(Cyrillic)", 1, n);
     n = index->getBucketIndex(zed, status);
@@ skipping 195 matching lines @@
     // ends up with a collator for the default locale
     // which makes the test unreliable. (see ticket #10277)
     // It exposes a bigger problem in that it may not be desirable for collation
     // to fall back to the default locale.
 
     // /* Basque*/ {"eu", "A:B:C:D:E:F:G:H:I:J:K:L:M:N:O:P:Q:R:S:T:U:V:W:X:Y:Z"},
 };
 
 void AlphabeticIndexTest::TestIndexCharactersList() {
     UErrorCode status = U_ZERO_ERROR;
-    for (int32_t i = 0; i < LENGTHOF(localeAndIndexCharactersLists); ++i) {
+    for (int32_t i = 0; i < UPRV_LENGTHOF(localeAndIndexCharactersLists); ++i) {
         const char *(&localeAndIndexCharacters)[2] = localeAndIndexCharactersLists[i];
         const char *locale = localeAndIndexCharacters[0];
         UnicodeString expectedIndexCharacters
             = (UnicodeString("\\u2026:") + localeAndIndexCharacters[1] + ":\\u2026").unescape();
         AlphabeticIndex index(locale, status);
         TEST_CHECK_STATUS;
         LocalPointer<AlphabeticIndex::ImmutableIndex> immIndex(index.buildImmutableIndex(status));
         TEST_CHECK_STATUS;
 
         // Join the elements of the list to a string with delimiter ":"
         UnicodeString actualIndexCharacters;
         assertEquals(locale,
                      expectedIndexCharacters,
                      joinLabelsAndAppend(*immIndex, actualIndexCharacters));
         logln(locale + UnicodeString(": ") + actualIndexCharacters);
     }
 }
 
 void AlphabeticIndexTest::TestHaniFirst() {
     UErrorCode status = U_ZERO_ERROR;
     LocalPointer<RuleBasedCollator> coll(
         static_cast<RuleBasedCollator *>(Collator::createInstance(Locale::getRoot(), status)));
 
     if (U_FAILURE(status)) {
         dataerrln("Failed Collator::createInstance call - %s", u_errorName(status));
         return;
     }
     int32_t reorderCodes[] = { USCRIPT_HAN };
-    coll->setReorderCodes(reorderCodes, LENGTHOF(reorderCodes), status);
+    coll->setReorderCodes(reorderCodes, UPRV_LENGTHOF(reorderCodes), status);
     TEST_CHECK_STATUS; 
     AlphabeticIndex index(coll.orphan(), status);
     TEST_CHECK_STATUS; 
     assertEquals("getBucketCount()", 1, index.getBucketCount(status));   // ... (underflow only)
     index.addLabels(Locale::getEnglish(), status);
     assertEquals("getBucketCount()", 28, index.getBucketCount(status));  // ... A-Z ...
     int32_t bucketIndex = index.getBucketIndex(UnicodeString((UChar)0x897f), status);
     assertEquals("getBucketIndex(U+897F)", 0, bucketIndex);  // underflow bucket
     bucketIndex = index.getBucketIndex("i", status);
     assertEquals("getBucketIndex(i)", 9, bucketIndex);
     bucketIndex = index.getBucketIndex(UnicodeString((UChar)0x03B1), status);
     assertEquals("getBucketIndex(Greek alpha)", 27, bucketIndex);
-    // TODO: Test with an unassigned code point (not just U+FFFF)
-    // when unassigned code points are not in the Hani reordering group any more.
-    // String unassigned = UTF16.valueOf(0x50005);
+    // U+50005 is an unassigned code point which sorts at the end, independent of the Hani group.
+    bucketIndex = index.getBucketIndex(UnicodeString(0x50005), status);
+    assertEquals("getBucketIndex(U+50005)", 27, bucketIndex);
     bucketIndex = index.getBucketIndex(UnicodeString((UChar)0xFFFF), status);
     assertEquals("getBucketIndex(U+FFFF)", 27, bucketIndex);
 }
 
 void AlphabeticIndexTest::TestPinyinFirst() {
     UErrorCode status = U_ZERO_ERROR;
     LocalPointer<RuleBasedCollator> coll(
         static_cast<RuleBasedCollator *>(Collator::createInstance(Locale::getChinese(), status)));
     if (U_FAILURE(status)) {
         dataerrln("Failed Collator::createInstance call - %s", u_errorName(status));
         return;
     }
     int32_t reorderCodes[] = { USCRIPT_HAN };
-    coll->setReorderCodes(reorderCodes, LENGTHOF(reorderCodes), status);
+    coll->setReorderCodes(reorderCodes, UPRV_LENGTHOF(reorderCodes), status);
     TEST_CHECK_STATUS;
     AlphabeticIndex index(coll.orphan(), status);
     TEST_CHECK_STATUS;
-    assertEquals("getBucketCount()", 1, index.getBucketCount(status));   // ... (underflow only)
+    assertEquals("getBucketCount()", 28, index.getBucketCount(status));   // ... A-Z ...
     index.addLabels(Locale::getChinese(), status);
     assertEquals("getBucketCount()", 28, index.getBucketCount(status));  // ... A-Z ...
     int32_t bucketIndex = index.getBucketIndex(UnicodeString((UChar)0x897f), status);
-    assertEquals("getBucketIndex(U+897F)", (int32_t)((UChar)0x0058/*X*/ - (UChar)0x0041/*A*/ + 1), (int32_t)bucketIndex);
+    assertEquals("getBucketIndex(U+897F)", (int32_t)((UChar)0x0058/*X*/ - (UChar)0x0041/*A*/ + 1), bucketIndex);
     bucketIndex = index.getBucketIndex("i", status);
     assertEquals("getBucketIndex(i)", 9, bucketIndex);
     bucketIndex = index.getBucketIndex(UnicodeString((UChar)0x03B1), status);
     assertEquals("getBucketIndex(Greek alpha)", (int32_t)27, bucketIndex);
-    // TODO: Test with an unassigned code point (not just U+FFFF)
-    // when unassigned code points are not in the Hani reordering group any more.
-    // String unassigned = UTF16.valueOf(0x50005);
+    // U+50005 is an unassigned code point which sorts at the end, independent of the Hani group.
+    bucketIndex = index.getBucketIndex(UnicodeString(0x50005), status);
+    assertEquals("getBucketIndex(U+50005)", 27, bucketIndex);
     bucketIndex = index.getBucketIndex(UnicodeString((UChar)0xFFFF), status);
     assertEquals("getBucketIndex(U+FFFF)", 27, bucketIndex);
 }
 
 void AlphabeticIndexTest::TestSchSt() {
     UErrorCode status = U_ZERO_ERROR;
     AlphabeticIndex index(Locale::getGerman(), status);
     index.addLabels(UnicodeSet("[\\u00C6{Sch*}{St*}]", status), status);
     TEST_CHECK_STATUS; 
     // ... A AE-ligature B-R S Sch St T-Z ...
@@ skipping 13 matching lines @@
         { "Aesthet", 2, "\\u00C6" },
         { "Berlin", 3, "B" },
         { "Rilke", 19, "R" },
         { "Sacher", 20, "S" },
         { "Seiler", 20, "S" },
         { "Sultan", 20, "S" },
         { "Schiller", 21, "Sch" },
         { "Steiff", 22, "St" },
         { "Thomas", 23, "T" }
     };
-    for (int32_t i = 0; i < LENGTHOF(testCases); ++i) {
+    for (int32_t i = 0; i < UPRV_LENGTHOF(testCases); ++i) {
         const TestCase &testCase = testCases[i];
         UnicodeString name = UnicodeString(testCase.name).unescape();
         UnicodeString label = UnicodeString(testCase.bucketLabel).unescape();
         char msg[100];
         sprintf(msg, "getBucketIndex(%s)", testCase.name);
         assertEquals(msg, testCase.bucketIndex, index.getBucketIndex(name, status));
         sprintf(msg, "immutable getBucketIndex(%s)", testCase.name);
         assertEquals(msg, testCase.bucketIndex, immIndex->getBucketIndex(name, status));
         sprintf(msg, "immutable bucket label (%s)", testCase.name);
         assertEquals(msg, label, immIndex->getBucket(testCase.bucketIndex)->getLabel());
@@ skipping 12 matching lines @@
     index.addRecord(UnicodeString((UChar)0x03B1), NULL, status);
     assertEquals("getBucketCount()", 1, index.getBucketCount(status));  // ...
     TEST_ASSERT(index.nextBucket(status));
     assertEquals("underflow label type", (int32_t)U_ALPHAINDEX_UNDERFLOW, index.getBucketLabelType());
     assertEquals("all records in the underflow bucket", (int32_t)3, index.getBucketRecordCount());
 }
 
 void AlphabeticIndexTest::TestChineseZhuyin() {
     UErrorCode status = U_ZERO_ERROR;
     char loc[100];
-    uloc_forLanguageTag("zh-u-co-zhuyin", loc, LENGTHOF(loc), NULL, &status);
+    uloc_forLanguageTag("zh-u-co-zhuyin", loc, UPRV_LENGTHOF(loc), NULL, &status);
     AlphabeticIndex index(loc, status);
     LocalPointer<AlphabeticIndex::ImmutableIndex> immIndex(index.buildImmutableIndex(status));
     TEST_CHECK_STATUS; 
     assertEquals("getBucketCount()", 38, immIndex->getBucketCount());
     assertEquals("label 1", UnicodeString((UChar)0x3105), immIndex->getBucket(1)->getLabel());
     assertEquals("label 2", UnicodeString((UChar)0x3106), immIndex->getBucket(2)->getLabel());
     assertEquals("label 3", UnicodeString((UChar)0x3107), immIndex->getBucket(3)->getLabel());
     assertEquals("label 4", UnicodeString((UChar)0x3108), immIndex->getBucket(4)->getLabel());
     assertEquals("label 5", UnicodeString((UChar)0x3109), immIndex->getBucket(5)->getLabel());
 }
 
+void AlphabeticIndexTest::TestJapaneseKanji() {
+    UErrorCode status = U_ZERO_ERROR;
+    AlphabeticIndex index(Locale::getJapanese(), status);
+    LocalPointer<AlphabeticIndex::ImmutableIndex> immIndex(index.buildImmutableIndex(status));
+    TEST_CHECK_STATUS;
+    // There are no index characters for Kanji in the Japanese standard collator.
+    // They should all go into the overflow bucket.
+    static const UChar32 kanji[] = { 0x4E9C, 0x95C7, 0x4E00, 0x58F1 };
+    int32_t overflowIndex = immIndex->getBucketCount() - 1;
+    for(int32_t i = 0; i < UPRV_LENGTHOF(kanji); ++i) {
+        char msg[40];
+        sprintf(msg, "kanji[%d]=U+%04lX in overflow bucket", (int)i, (long)kanji[i]);
+        assertEquals(msg, overflowIndex, immIndex->getBucketIndex(UnicodeString(kanji[i]), status));
+        TEST_CHECK_STATUS;
+    }
+}
+
+void AlphabeticIndexTest::TestChineseUnihan() {
+    UErrorCode status = U_ZERO_ERROR;
+    AlphabeticIndex index("zh-u-co-unihan", status);
+    if(U_FAILURE(status)) {
+        dataerrln("unable create an AlphabeticIndex for Chinese/unihan: %s", u_errorName(status));
+        return;
+    }
+    index.setMaxLabelCount(500, status);  // ICU 54 default is 99.
+    LocalPointer<AlphabeticIndex::ImmutableIndex> immIndex(index.buildImmutableIndex(status));
+    TEST_CHECK_STATUS;
+    int32_t bucketCount = immIndex->getBucketCount();
+    if(bucketCount < 216) {
+        // There should be at least an underflow and overflow label,
+        // and one for each of 214 radicals,
+        // and maybe additional labels for simplified radicals.
+        dataerrln("too few buckets/labels for Chinese/unihan: %d (is zh/unihan data available?)",
+                  bucketCount);
+        return;
+    } else {
+        logln("Chinese/unihan has %d buckets/labels", bucketCount);
+    }
+    // bucketIndex = radical number, adjusted for simplified radicals in lower buckets.
+    int32_t bucketIndex = index.getBucketIndex(UnicodeString((UChar)0x4e5d), status);
+    assertEquals("getBucketIndex(U+4E5D)", 5, bucketIndex);
+    bucketIndex = index.getBucketIndex(UnicodeString((UChar)0x7527), status);
+    assertEquals("getBucketIndex(U+7527)", 100, bucketIndex);
+}
+
 #endif