Update ICU to 54.1 step 1

source/test/cintltst/capitst.c

 /********************************************************************
- * Copyright (c) 1997-2013, International Business Machines
+ * Copyright (c) 1997-2014, International Business Machines
  * Corporation and others. All Rights Reserved.
  ********************************************************************/
 /*****************************************************************************
 *
 * File CAPITEST.C
 *
 * Modification History:
 *        Name                     Description
 *     Madhu Katragadda             Ported for C API
 *     Brian Rower                  Added TestOpenVsOpenRules
@@ skipping 11 matching lines @@
 #include "unicode/ulocdata.h"
 #include "unicode/ustring.h"
 #include "unicode/ures.h"
 #include "unicode/ucoleitr.h"
 #include "cintltst.h"
 #include "capitst.h"
 #include "ccolltst.h"
 #include "putilimp.h"
 #include "cmemory.h"
 #include "cstring.h"
+#include "ucol_imp.h"
 
 static void TestAttribute(void);
 static void TestDefault(void);
 static void TestDefaultKeyword(void);
 static void TestBengaliSortKey(void);
 
 
 static char* U_EXPORT2 ucol_sortKeyToString(const UCollator *coll, const uint8_t *sortkey, char *buffer, uint32_t len) {
     uint32_t position = 0;
     uint8_t b;
@@ skipping 216 matching lines @@
 
 }
 #endif
 
 /* Collator Properties
  ucol_open, ucol_strcoll,  getStrength/setStrength
  getDecomposition/setDecomposition, getDisplayName*/
 void TestProperty()
 {
     UCollator *col, *ruled;
+    const UChar *rules;
     UChar *disName;
     int32_t len = 0;
     UChar source[12], target[12];
     int32_t tempLength;
     UErrorCode status = U_ZERO_ERROR;
     /*
      * Expected version of the English collator.
      * Currently, the major/minor version numbers change when the builder code
      * changes,
      * number 2 is from the tailoring data version and
@@ skipping 23 matching lines @@
               versionArray[0], versionArray[1], versionArray[2], versionArray[3]);
     } else {
       log_verbose("ucol_getVersion() result: %02x.%02x.%02x.%02x\n",
                   versionArray[0], versionArray[1], versionArray[2], versionArray[3]);
     }
 
     /* Assume that the UCD and UCA versions are the same,
      * rather than hardcoding (and updating each time) a particular UCA version. */
     u_getUnicodeVersion(versionUCDArray);
     ucol_getUCAVersion(col, versionUCAArray);
-    if (0!=uprv_memcmp(versionUCAArray, versionUCDArray, 4) /*TODO: remove the following once UCA is up to date, ticket:9101*/ && !(versionUCDArray[0]==6 && versionUCDArray[1]==3)) {
+    if (0!=uprv_memcmp(versionUCAArray, versionUCDArray, 4)) {
       log_err("Testing ucol_getUCAVersion() - unexpected result: %hu.%hu.%hu.%hu\n",
               versionUCAArray[0], versionUCAArray[1], versionUCAArray[2], versionUCAArray[3]);
     }
 
     u_uastrcpy(source, "ab");
     u_uastrcpy(target, "abc");
 
     doStrcoll(col, source, u_strlen(source), target, u_strlen(target), UCOL_LESS, "ab < abc comparison failed");
 
     u_uastrcpy(source, "ab");
@@ skipping 39 matching lines @@
         log_verbose("the display name for default collation in german: %s\n", austrdup(disName) );
         free(disName);
     }
     if(U_FAILURE(status)){
         log_err("ERROR: in getDisplayName: %s\n", myErrorName(status));
         return;
     }
     log_verbose("Default collation getDisplayName ended.\n");
 
     ruled = ucol_open("da_DK", &status);
+    if(U_FAILURE(status)) {
+        log_data_err("ucol_open(\"da_DK\") failed - %s\n", u_errorName(status));
+        ucol_close(col);
+        return;
+    }
     log_verbose("ucol_getRules() testing ...\n");
-    ucol_getRules(ruled, &tempLength);
-    doAssert( tempLength != 0, "getRules() result incorrect" );
+    rules = ucol_getRules(ruled, &tempLength);
+    if(tempLength == 0) {
+        log_data_err("missing da_DK tailoring rule string\n");
+    } else {
+        UChar aa[2] = { 0x61, 0x61 };
+        doAssert(u_strFindFirst(rules, tempLength, aa, 2) != NULL,
+                 "da_DK rules do not contain 'aa'");
+    }
     log_verbose("getRules tests end.\n");
     {
         UChar *buffer = (UChar *)malloc(200000*sizeof(UChar));
         int32_t bufLen = 200000;
         buffer[0] = '\0';
         log_verbose("ucol_getRulesEx() testing ...\n");
         tempLength = ucol_getRulesEx(col,UCOL_TAILORING_ONLY,buffer,bufLen );
         doAssert( tempLength == 0x00, "getRulesEx() result incorrect" );
         log_verbose("getRules tests end.\n");
 
         log_verbose("ucol_getRulesEx() testing ...\n");
         tempLength=ucol_getRulesEx(col,UCOL_FULL_RULES,buffer,bufLen );
-        doAssert( tempLength != 0, "getRulesEx() result incorrect" );
-        log_verbose("getRules tests end.\n");
+        if(tempLength == 0) {
+            log_data_err("missing *full* rule string\n");
+        }
+        log_verbose("getRulesEx tests end.\n");
         free(buffer);
     }
     ucol_close(ruled);
     ucol_close(col);
 
     log_verbose("open an collator for french locale");
     col = ucol_open("fr_FR", &status);
     if (U_FAILURE(status)) {
        log_err("ERROR: Creating French collation failed.: %s\n", myErrorName(status));
         return;
@@ skipping 28 matching lines @@
 }
 
 /* Test RuleBasedCollator and getRules*/
 void TestRuleBasedColl()
 {
     UCollator *col1, *col2, *col3, *col4;
     UCollationElements *iter1, *iter2;
     UChar ruleset1[60];
     UChar ruleset2[50];
     UChar teststr[10];
-    UChar teststr2[10];
     const UChar *rule1, *rule2, *rule3, *rule4;
     int32_t tempLength;
     UErrorCode status = U_ZERO_ERROR;
     u_uastrcpy(ruleset1, "&9 < a, A < b, B < c, C; ch, cH, Ch, CH < d, D, e, E");
     u_uastrcpy(ruleset2, "&9 < a, A < b, B < c, C < d, D, e, E");
 
 
     col1 = ucol_openRules(ruleset1, u_strlen(ruleset1), UCOL_DEFAULT, UCOL_DEFAULT_STRENGTH, NULL,&status);
     if (U_FAILURE(status)) {
         log_err_status(status, "RuleBased Collator creation failed.: %s\n", myErrorName(status));
@@ skipping 75 matching lines @@
              log_err("! modifier test failed");
         }
         if (ce == UCOL_NULLORDER) {
             break;
         }
     }
     ucol_closeElements(iter1);
     ucol_closeElements(iter2);
     ucol_close(col1);
     ucol_close(col2);
-    /* test that we can start a rule without a & or < */
+    /* CLDR 24+ requires a reset before the first relation */
     u_uastrcpy(ruleset1, "< z < a");
     col1 = ucol_openRules(ruleset1, u_strlen(ruleset1), UCOL_DEFAULT, UCOL_DEFAULT_STRENGTH, NULL, &status);
-    if (U_FAILURE(status)) {
-        log_err("RuleBased Collator creation failed.: %s\n", myErrorName(status));
-        return;
-    }
-    u_uastrcpy(teststr, "z");
-    u_uastrcpy(teststr2, "a");
-    if (ucol_greaterOrEqual(col1, teststr, 1, teststr2, 1)) {
-        log_err("Rule \"z < a\" fails");
+    if (status != U_PARSE_ERROR && status != U_INVALID_FORMAT_ERROR) {
+        log_err("ucol_openRules(without initial reset: '< z < a') "
+                "should fail with U_PARSE_ERROR or U_INVALID_FORMAT_ERROR but yielded %s\n",
+                myErrorName(status));
     }
     ucol_close(col1);
 }
 
 void TestCompare()
 {
     UErrorCode status = U_ZERO_ERROR;
     UCollator *col;
     UChar* test1;
     UChar* test2;
@@ skipping 404 matching lines @@
 
         /* create a collator the normal API way */
         c1 = ucol_open(curLoc, &err);
         if (U_FAILURE(err)) {
             log_err("ERROR: Normal collation creation failed with locale: %s : %s\n", curLoc, myErrorName(err));
             return;
         }
 
         /* grab the rules */
         rules = ucol_getRules(c1, &rulesLength);
+        if (rulesLength == 0) {
+            /* The optional tailoring rule string is either empty (boring) or missing. */
+            ucol_close(c1);
+            continue;
+        }
 
         /* use those rules to create a collator from rules */
         c2 = ucol_openRules(rules, rulesLength, UCOL_DEFAULT, UCOL_DEFAULT_STRENGTH, NULL, &err);
         if (U_FAILURE(err)) {
             log_err("ERROR: Creating collator from rules failed with locale: %s : %s\n", curLoc, myErrorName(err));
-            return;
+            ucol_close(c1);
+            continue;
         }
 
         uld = ulocdata_open(curLoc, &err);
 
         /*now that we have some collators, we get several strings */
 
         for(y = 0; y < 5; y++){
 
             /* get a set of ALL the characters in this locale */
             eSet =  ulocdata_getExemplarSet(uld, NULL, 0, ULOCDATA_ES_STANDARD, &err);
@@ skipping 187 matching lines @@
     ucol_setAttribute(col, UCOL_STRENGTH, UCOL_TERTIARY, &status);
     sortklen = ucol_getSortKey(col, test1, 0, NULL, 0);
     sortkEmpty = (uint8_t*)malloc(sizeof(uint8_t) * sortklen+1);
     sortklen = ucol_getSortKey(col, test1, 0, sortkEmpty, sortklen+1);
     if(sortklen != 3 || sortkEmpty[0] != 1 || sortkEmpty[0] != 1 || sortkEmpty[2] != 0) {
       log_err("Empty string generated wrong sortkey!\n");
     }
     free(sortkEmpty);
 
     log_verbose("testing passing invalid string\n");
-    sortklen = ucol_getSortKey(col, NULL, 0, NULL, 0);
+    sortklen = ucol_getSortKey(col, NULL, 10, NULL, 0);
     if(sortklen != 0) {
       log_err("Invalid string didn't return sortkey size of 0\n");
     }
 
 
     log_verbose("testing sortkey ends...\n");
     ucol_close(col);
     free(test1);
     free(test2);
     free(test3);
@@ skipping 189 matching lines @@
   UErrorCode status = U_ZERO_ERROR;
   const char *rules = "&a<x<y<z";
   UChar rlz[256] = {0};
   uint32_t rlzLen = u_unescape(rules, rlz, 256);
 
   UCollator *coll = NULL;
   const char *locale = NULL;
 
   int32_t i = 0;
 
-  /* Now that the collation tree is separate, actual==valid at all times. [alan] */
   static const struct {
     const char* requestedLocale;
     const char* validLocale;
     const char* actualLocale;
   } testStruct[] = {
     { "sr_RS", "sr_Cyrl_RS", "sr" },
     { "sh_YU", "sr_Latn_RS", "sr_Latn" }, /* was sh, then aliased to hr, now sr_Latn via import per cldrbug 5647: */
-    { "en_BE_FOO", "en_BE", "root" },
-    { "sv_SE_NONEXISTANT", "sv_SE", "sv" }
+    { "en_BE_FOO", "en", "root" },
+    { "sv_SE_NONEXISTANT", "sv", "sv" }
   };
 
   /* test opening collators for different locales */
   for(i = 0; i<sizeof(testStruct)/sizeof(testStruct[0]); i++) {
     status = U_ZERO_ERROR;
     coll = ucol_open(testStruct[i].requestedLocale, &status);
     if(U_FAILURE(status)) {
       log_err_status(status, "Failed to open collator for %s with %s\n", testStruct[i].requestedLocale, u_errorName(status));
       ucol_close(coll);
       continue;
     }
-   locale = ucol_getLocaleByType(coll, ULOC_REQUESTED_LOCALE, &status);
-    if(strcmp(locale, testStruct[i].requestedLocale) != 0) {
+    /*
+     * The requested locale may be the same as the valid locale,
+     * or may not be supported at all. See ticket #10477.
+     */
+    locale = ucol_getLocaleByType(coll, ULOC_REQUESTED_LOCALE, &status);
+    if(U_SUCCESS(status) &&
+          strcmp(locale, testStruct[i].requestedLocale) != 0 && strcmp(locale, testStruct[i].validLocale) != 0) {
       log_err("[Coll %s]: Error in requested locale, expected %s, got %s\n", testStruct[i].requestedLocale, testStruct[i].requestedLocale, locale);
     }
+    status = U_ZERO_ERROR;
     locale = ucol_getLocaleByType(coll, ULOC_VALID_LOCALE, &status);
     if(strcmp(locale, testStruct[i].validLocale) != 0) {
       log_err("[Coll %s]: Error in valid locale, expected %s, got %s\n", testStruct[i].requestedLocale, testStruct[i].validLocale, locale);
     }
     locale = ucol_getLocaleByType(coll, ULOC_ACTUAL_LOCALE, &status);
     if(strcmp(locale, testStruct[i].actualLocale) != 0) {
       log_err("[Coll %s]: Error in actual locale, expected %s, got %s\n", testStruct[i].requestedLocale, testStruct[i].actualLocale, locale);
     }
     ucol_close(coll);
   }
 
   /* completely non-existant locale for collator should get a default collator */
   {
     UCollator *defaultColl = ucol_open(NULL, &status);
     coll = ucol_open("blahaha", &status);
     if(U_SUCCESS(status)) {
+      /* See comment above about ticket #10477.
       if(strcmp(ucol_getLocaleByType(coll, ULOC_REQUESTED_LOCALE, &status), "blahaha")) {
         log_err("Nonexisting locale didn't preserve the requested locale\n");
-      }
+      } */
       if(strcmp(ucol_getLocaleByType(coll, ULOC_VALID_LOCALE, &status),
         ucol_getLocaleByType(defaultColl, ULOC_VALID_LOCALE, &status))) {
         log_err("Valid locale for nonexisting locale locale collator differs "
           "from valid locale for default collator\n");
       }
       if(strcmp(ucol_getLocaleByType(coll, ULOC_ACTUAL_LOCALE, &status),
         ucol_getLocaleByType(defaultColl, ULOC_ACTUAL_LOCALE, &status))) {
         log_err("Actual locale for nonexisting locale locale collator differs "
           "from actual locale for default collator\n");
       }
       ucol_close(coll);
       ucol_close(defaultColl);
     } else {
       log_data_err("Couldn't open collators\n");
     }
   }
 
 
 
   /* collator instantiated from rules should have all three locales NULL */
   coll = ucol_openRules(rlz, rlzLen, UCOL_DEFAULT, UCOL_DEFAULT, NULL, &status);
-  locale = ucol_getLocaleByType(coll, ULOC_REQUESTED_LOCALE, &status);
-  if(locale != NULL) {
-    log_err("For collator instantiated from rules, requested locale returned %s instead of NULL\n", locale);
+  if (coll != NULL) {
+    locale = ucol_getLocaleByType(coll, ULOC_REQUESTED_LOCALE, &status);
+    if(U_SUCCESS(status) && locale != NULL) {
+      log_err("For collator instantiated from rules, requested locale returned %s instead of NULL\n", locale);
+    }
+    status = U_ZERO_ERROR;
+    locale = ucol_getLocaleByType(coll, ULOC_VALID_LOCALE, &status);
+    if(locale != NULL) {
+      log_err("For collator instantiated from rules,  valid locale returned %s instead of NULL\n", locale);
+    }
+    locale = ucol_getLocaleByType(coll, ULOC_ACTUAL_LOCALE, &status);
+    if(locale != NULL) {
+      log_err("For collator instantiated from rules, actual locale returned %s instead of NULL\n", locale);
+    }
+    ucol_close(coll);
+  } else {
+    log_data_err("Couldn't get collator from ucol_openRules() - %s\n", u_errorName(status));
   }
-  locale = ucol_getLocaleByType(coll, ULOC_VALID_LOCALE, &status);
-  if(locale != NULL) {
-    log_err("For collator instantiated from rules,  valid locale returned %s instead of NULL\n", locale);
-  }
-  locale = ucol_getLocaleByType(coll, ULOC_ACTUAL_LOCALE, &status);
-  if(locale != NULL) {
-    log_err("For collator instantiated from rules, actual locale returned %s instead of NULL\n", locale);
-  }
-  ucol_close(coll);
-
 }
 
 
 void TestGetAll()
 {
     int32_t i, count;
     count=ucol_countAvailable();
     /* use something sensible w/o hardcoding the count */
     if(count < 0){
         log_err("Error in countAvailable(), it returned %d\n", count);
@@ skipping 347 matching lines @@
 
   int32_t i = 0, j = 0;
   UErrorCode status = U_ZERO_ERROR;
   UParseError pError;
 
   UCollator *coll = NULL;
   UChar buff[1024];
   int32_t buffLen = 0;
   USet *set = NULL;
 
-  for(i = 0; i < sizeof(setTest)/sizeof(setTest[0]); i++) {
+  for(i = 0; i < UPRV_LENGTHOF(setTest); i++) {
     buffLen = u_unescape(setTest[i].rules, buff, 1024);
     coll = ucol_openRules(buff, buffLen, UCOL_DEFAULT, UCOL_DEFAULT, &pError, &status);
     if(U_SUCCESS(status)) {
       set = ucol_getTailoredSet(coll, &status);
-      if(uset_size(set) != setTest[i].testsize) {
-        log_err("Tailored set size different (%d) than expected (%d)\n", uset_size(set), setTest[i].testsize);
+      if(uset_size(set) < setTest[i].testsize) {
+        log_err("Tailored set size smaller (%d) than expected (%d)\n", uset_size(set), setTest[i].testsize);
       }
       for(j = 0; j < setTest[i].testsize; j++) {
         buffLen = u_unescape(setTest[i].tests[j], buff, 1024);
         if(!uset_containsString(set, buff, buffLen)) {
           log_err("Tailored set doesn't contain %s... It should\n", setTest[i].tests[j]);
         }
       }
       uset_close(set);
     } else {
       log_err_status(status, "Couldn't open collator with rules %s\n", setTest[i].rules);
@@ skipping 142 matching lines @@
 {
     struct {
         const char *input;
         const char *expectedOutput;
         const char *locale;
         UErrorCode expectedStatus;
         int32_t    expectedOffset;
         uint32_t   expectedIdentifier;
     } testCases[] = {
         /*
-         * The following expectedOutput contains a collation weight (2700 from UCA 6.0)
-         * which is the primary weight for the T character (U+0041) in the input.
-         * When that character gets a different weight in FractionalUCA.txt,
-         * the expectedOutput needs to be adjusted.
-         * That is, when we upgrade to a new UCA version or change collation
-         * in such a way that the absolute weight for 'A' changes,
-         * we will get a test failure here and need to adjust the test case.
+         * Note: The first test case sets variableTop to the dollar sign '$'.
+         * We have agreed to drop support for variableTop in ucol_getShortDefinitionString(),
+         * related to ticket #10372 "deprecate collation APIs for short definition strings",
+         * and because it did not work for most spaces/punctuation/symbols,
+         * as documented in ticket #10386 "collation short definition strings issues":
+         * The old code wrote only 3 hex digits for primary weights below 0x0FFF,
+         * which is a syntax error, and then failed to normalize the result.
+         *
+         * The "B2700" was removed from the expected result ("B2700_KPHONEBOOK_LDE").
+         *
+         * Previously, this test had to be adjusted for root collator changes because the
+         * primary weight of the variable top character naturally changed
+         * but was baked into the expected result.
          */
-        {"LDE_RDE_KPHONEBOOK_T0041_ZLATN","B2700_KPHONEBOOK_LDE", "de@collation=phonebook", U_USING_FALLBACK_WARNING, 0, 0 },
+        {"LDE_RDE_KPHONEBOOK_T0024_ZLATN","KPHONEBOOK_LDE", "de@collation=phonebook", U_USING_FALLBACK_WARNING, 0, 0 },
 
         {"LEN_RUS_NO_AS_S4","AS_LROOT_NO_S4", NULL, U_USING_DEFAULT_WARNING, 0, 0 },
         {"LDE_VPHONEBOOK_EO_SI","EO_KPHONEBOOK_LDE_SI", "de@collation=phonebook", U_ZERO_ERROR, 0, 0 },
         {"LDE_Kphonebook","KPHONEBOOK_LDE", "de@collation=phonebook", U_ZERO_ERROR, 0, 0 },
         {"Xqde_DE@collation=phonebookq_S3_EX","KPHONEBOOK_LDE", "de@collation=phonebook", U_USING_FALLBACK_WARNING, 0, 0 },
         {"LFR_FO", "FO_LROOT", NULL, U_USING_DEFAULT_WARNING, 0, 0 },
         {"SO_LX_AS", "", NULL, U_ILLEGAL_ARGUMENT_ERROR, 8, 0 },
         {"S3_ASS_MMM", "", NULL, U_ILLEGAL_ARGUMENT_ERROR, 5, 0 }
     };
 
@@ skipping 51 matching lines @@
                 log_err("Got parse error offset %i, but expected %i instead for '%s'\n",
                     parseError.offset, testCases[i].expectedOffset, testCases[i].input);
             }
         }
     }
 
 }
 
 static void
 doSetsTest(const char *locale, const USet *ref, USet *set, const char* inSet, const char* outSet, UErrorCode *status) {
-    UChar buffer[512];
+    UChar buffer[65536];
     int32_t bufLen;
 
     uset_clear(set);
     bufLen = u_unescape(inSet, buffer, 512);
     uset_applyPattern(set, buffer, bufLen, 0, status);
     if(U_FAILURE(*status)) {
         log_err("%s: Failure setting pattern %s\n", locale, u_errorName(*status));
     }
 
     if(!uset_containsAll(ref, set)) {
         log_err("%s: Some stuff from %s is not present in the set\n", locale, inSet);
+        uset_removeAll(set, ref);
+        bufLen = uset_toPattern(set, buffer, UPRV_LENGTHOF(buffer), TRUE, status);
+        log_info("    missing: %s\n", aescstrdup(buffer, bufLen));
+        bufLen = uset_toPattern(ref, buffer, UPRV_LENGTHOF(buffer), TRUE, status);
+        log_info("    total: size=%i  %s\n", uset_getItemCount(ref), aescstrdup(buffer, bufLen));
     }
 
     uset_clear(set);
     bufLen = u_unescape(outSet, buffer, 512);
     uset_applyPattern(set, buffer, bufLen, 0, status);
     if(U_FAILURE(*status)) {
         log_err("%s: Failure setting pattern %s\n", locale, u_errorName(*status));
     }
 
     if(!uset_containsNone(ref, set)) {
@@ skipping 34 matching lines @@
         },
         { "sh",
             "[{C\\u0301}{C\\u030C}{C\\u0341}{DZ\\u030C}{Dz\\u030C}{D\\u017D}{D\\u017E}{lj}{nj}]",
             "[{\\u309d\\u3099}{\\u30fd\\u3099}]",
             "[\\u00e6]",
             "[a]",
             "[nlcdzNLCDZ]",
             "[jabv]"
         },
         { "ja",
-          "[{\\u3053\\u3099\\u309D}{\\u3053\\u3099\\u309D\\u3099}{\\u3053\\u3099\\u309E}{\\u3053\\u3099\\u30FC}{\\u3053\\u309D}{\\u3053\\u309D\\u3099}{\\u3053\\u309E}{\\u3053\\u30FC}{\\u30B3\\u3099\\u30FC}{\\u30B3\\u3099\\u30FD}{\\u30B3\\u3099\\u30FD\\u3099}{\\u30B3\\u3099\\u30FE}{\\u30B3\\u30FC}{\\u30B3\\u30FD}{\\u30B3\\u30FD\\u3099}{\\u30B3\\u30FE}]",
+          /*
+           * The "collv2" builder omits mappings if the collator maps their
+           * character sequences to the same CEs.
+           * For example, it omits Japanese contractions for NFD forms
+           * of the voiced iteration mark (U+309E = U+309D + U+3099), such as
+           * {\\u3053\\u3099\\u309D\\u3099}{\\u3053\\u309D\\u3099}
+           * {\\u30B3\\u3099\\u30FD\\u3099}{\\u30B3\\u30FD\\u3099}.
+           * It does add mappings for the precomposed forms.
+           */
+          "[{\\u3053\\u3099\\u309D}{\\u3053\\u3099\\u309E}{\\u3053\\u3099\\u30FC}"
+           "{\\u3053\\u309D}{\\u3053\\u309E}{\\u3053\\u30FC}"
+           "{\\u30B3\\u3099\\u30FC}{\\u30B3\\u3099\\u30FD}{\\u30B3\\u3099\\u30FE}"
+           "{\\u30B3\\u30FC}{\\u30B3\\u30FD}{\\u30B3\\u30FE}]",
           "[{\\u30FD\\u3099}{\\u309D\\u3099}{\\u3053\\u3099}{\\u30B3\\u3099}{lj}{nj}]",
             "[\\u30FE\\u00e6]",
             "[a]",
             "[\\u3099]",
             "[]"
         }
     };
 
-
-
-
     UErrorCode status = U_ZERO_ERROR;
     UCollator *coll = NULL;
     int32_t i = 0;
     int32_t noConts = 0;
     USet *conts = uset_open(0,0);
     USet *exp = uset_open(0, 0);
     USet *set  = uset_open(0,0);
     int32_t setBufferLen = 65536;
     UChar buffer[65536];
     int32_t setLen = 0;
@@ skipping 39 matching lines @@
 
 
     uset_close(conts);
     uset_close(exp);
     uset_close(set);
 }
 
 static void
 TestOpenBinary(void)
 {
+    /*
+     * ucol_openBinary() documents:
+     * "The API also takes a base collator which usually should be UCA."
+     * and
+     * "Currently it cannot be NULL."
+     *
+     * However, the check for NULL was commented out in ICU 3.4 (r18149).
+     * Ticket #4355 requested "Make collation work with minimal data.
+     * Optionally without UCA, with relevant parts of UCA copied into the tailoring table."
+     *
+     * The ICU team agreed with ticket #10517 "require base collator in ucol_openBinary() etc."
+     * to require base!=NULL again.
+     */
+#define OPEN_BINARY_ACCEPTS_NULL_BASE 0
     UErrorCode status = U_ZERO_ERROR;
     /*
     char rule[] = "&h < d < c < b";
     char *wUCA[] = { "a", "h", "d", "c", "b", "i" };
     char *noUCA[] = {"d", "c", "b", "a", "h", "i" };
     */
     /* we have to use Cyrillic letters because latin-1 always gets copied */
     const char rule[] = "&\\u0452 < \\u0434 < \\u0433 < \\u0432"; /* &dje < d < g < v */
     const char *wUCA[] = { "\\u0430", "\\u0452", "\\u0434", "\\u0433", "\\u0432", "\\u0435" }; /* a, dje, d, g, v, e */
+#if OPEN_BINARY_ACCEPTS_NULL_BASE
     const char *noUCA[] = {"\\u0434", "\\u0433", "\\u0432", "\\u0430", "\\u0435", "\\u0452" }; /* d, g, v, a, e, dje */
+#endif
 
     UChar uRules[256];
     int32_t uRulesLen = u_unescape(rule, uRules, 256);
 
     UCollator *coll = ucol_openRules(uRules, uRulesLen, UCOL_DEFAULT, UCOL_DEFAULT, NULL, &status);
     UCollator *UCA = NULL;
     UCollator *cloneNOUCA = NULL, *cloneWUCA = NULL;
 
     uint8_t imageBuffer[32768];
     uint8_t *image = imageBuffer;
@@ skipping 15 matching lines @@
     imageSize = ucol_cloneBinary(coll, image, imageBufferCapacity, &status);
     if(U_FAILURE(status)) {
         image = (uint8_t *)malloc(imageSize*sizeof(uint8_t));
         status = U_ZERO_ERROR;
         imageSize = ucol_cloneBinary(coll, imageBuffer, imageSize, &status);
     }
 
 
     cloneWUCA = ucol_openBinary(image, imageSize, UCA, &status);
     cloneNOUCA = ucol_openBinary(image, imageSize, NULL, &status);
+#if !OPEN_BINARY_ACCEPTS_NULL_BASE
+    if(status != U_ILLEGAL_ARGUMENT_ERROR) {
+        log_err("ucol_openBinary(base=NULL) unexpectedly did not fail - %s\n", u_errorName(status));
+    }
+#endif
 
     genericOrderingTest(coll, wUCA, sizeof(wUCA)/sizeof(wUCA[0]));
 
     genericOrderingTest(cloneWUCA, wUCA, sizeof(wUCA)/sizeof(wUCA[0]));
+#if OPEN_BINARY_ACCEPTS_NULL_BASE
     genericOrderingTest(cloneNOUCA, noUCA, sizeof(noUCA)/sizeof(noUCA[0]));
+#endif
 
     if(image != imageBuffer) {
         free(image);
     }
     ucol_close(coll);
     ucol_close(cloneNOUCA);
     ucol_close(cloneWUCA);
     ucol_close(UCA);
 }
 
@@ skipping 30 matching lines @@
         log_info("Warning: ucol_open(%s, ...) returned %s, at least it didn't crash.\n", loc, u_errorName(status));
     } else if (status != U_USING_FALLBACK_WARNING) {
         /* Hmm, skip the following test for CLDR 1.9 data and/or ICU 4.6, no longer seems to apply */
         #if 0
         log_err("ucol_open(%s, ...) should return an error or some sort of U_USING_FALLBACK_WARNING, but returned %s\n", loc, u_errorName(status));
         #endif
     }
     ucol_close(coll);
 }
 
+static UBool uenum_contains(UEnumeration *e, const char *s, UErrorCode *status) {
+    const char *t;
+    uenum_reset(e, status);
+    while(((t = uenum_next(e, NULL, status)) != NULL) && U_SUCCESS(*status)) {
+        if(uprv_strcmp(s, t) == 0) {
+            return TRUE;
+        }
+    }
+    return FALSE;
+}
+
 static void TestGetKeywordValuesForLocale(void) {
-#define INCLUDE_UNIHAN_COLLATION 0
-#define PREFERRED_SIZE 16
 #define MAX_NUMBER_OF_KEYWORDS 9
-    const char *PREFERRED[PREFERRED_SIZE][MAX_NUMBER_OF_KEYWORDS+1] = {
+    const char *PREFERRED[][MAX_NUMBER_OF_KEYWORDS+1] = {
             { "und",            "standard", "eor", "search", NULL, NULL, NULL, NULL, NULL, NULL },
             { "en_US",          "standard", "eor", "search", NULL, NULL, NULL, NULL, NULL, NULL },
             { "en_029",         "standard", "eor", "search", NULL, NULL, NULL, NULL, NULL, NULL },
             { "de_DE",          "standard", "phonebook", "search", "eor", NULL, NULL, NULL, NULL, NULL },
             { "de_Latn_DE",     "standard", "phonebook", "search", "eor", NULL, NULL, NULL, NULL, NULL },
-#if INCLUDE_UNIHAN_COLLATION
-            { "zh",             "pinyin", "big5han", "gb2312han", "stroke", "unihan", "zhuyin", "eor", "search", "standard" },
-            { "zh_Hans",        "pinyin", "big5han", "gb2312han", "stroke", "unihan", "zhuyin", "eor", "search", "standard" },
-            { "zh_CN",          "pinyin", "big5han", "gb2312han", "stroke", "unihan", "zhuyin", "eor", "search", "standard" },
-            { "zh_Hant",        "stroke", "big5han", "gb2312han", "pinyin", "unihan", "zhuyin", "eor", "search", "standard" },
-            { "zh_TW",          "stroke", "big5han", "gb2312han", "pinyin", "unihan", "zhuyin", "eor", "search", "standard" },
-            { "zh__PINYIN",     "pinyin", "big5han", "gb2312han", "stroke", "unihan", "zhuyin", "eor", "search", "standard" },
-#else
-            { "zh",             "pinyin", "big5han", "gb2312han", "stroke", "zhuyin", "eor", "search", "standard", NULL },
-            { "zh_Hans",        "pinyin", "big5han", "gb2312han", "stroke", "zhuyin", "eor", "search", "standard", NULL },
-            { "zh_CN",          "pinyin", "big5han", "gb2312han", "stroke", "zhuyin", "eor", "search", "standard", NULL },
-            { "zh_Hant",        "stroke", "big5han", "gb2312han", "pinyin", "zhuyin", "eor", "search", "standard", NULL },
-            { "zh_TW",          "stroke", "big5han", "gb2312han", "pinyin", "zhuyin", "eor", "search", "standard", NULL },
-            { "zh__PINYIN",     "pinyin", "big5han", "gb2312han", "stroke", "zhuyin", "eor", "search", "standard", NULL },
-#endif
+            { "zh",             "pinyin", "stroke", "eor", "search", "standard", NULL },
+            { "zh_Hans",        "pinyin", "stroke", "eor", "search", "standard", NULL },
+            { "zh_CN",          "pinyin", "stroke", "eor", "search", "standard", NULL },
+            { "zh_Hant",        "stroke", "pinyin", "eor", "search", "standard", NULL },
+            { "zh_TW",          "stroke", "pinyin", "eor", "search", "standard", NULL },
+            { "zh__PINYIN",     "pinyin", "stroke", "eor", "search", "standard", NULL },
             { "es_ES",          "standard", "search", "traditional", "eor", NULL, NULL, NULL, NULL, NULL },
             { "es__TRADITIONAL","traditional", "search", "standard", "eor", NULL, NULL, NULL, NULL, NULL },
             { "und@collation=phonebook",    "standard", "eor", "search", NULL, NULL, NULL, NULL, NULL, NULL },
-            { "de_DE@collation=big5han",    "standard", "phonebook", "search", "eor", NULL, NULL, NULL, NULL, NULL },
+            { "de_DE@collation=pinyin",     "standard", "phonebook", "search", "eor", NULL, NULL, NULL, NULL, NULL },
             { "zzz@collation=xxx",          "standard", "eor", "search", NULL, NULL, NULL, NULL, NULL, NULL }
     };
-#if INCLUDE_UNIHAN_COLLATION
-    const int32_t expectedLength[PREFERRED_SIZE] = { 3, 3, 3, 4, 4, 9, 9, 9, 9, 9, 9, 4, 4, 3, 4, 3 };
-#else
-    const int32_t expectedLength[PREFERRED_SIZE] = { 3, 3, 3, 4, 4, 8, 8, 8, 8, 8, 8, 4, 4, 3, 4, 3 };
-#endif
 
     UErrorCode status = U_ZERO_ERROR;
     UEnumeration *keywordValues = NULL;
     int32_t i, n, size, valueLength;
     const char *locale = NULL, *value = NULL;
     UBool errorOccurred = FALSE;
 
-    for (i = 0; i < PREFERRED_SIZE; i++) {
+    for (i = 0; i < UPRV_LENGTHOF(PREFERRED) && !errorOccurred; i++) {
         locale = PREFERRED[i][0];
         value = NULL;
         valueLength = 0;
         size = 0;
 
         keywordValues = ucol_getKeywordValuesForLocale("collation", locale, TRUE, &status);
         if (keywordValues == NULL || U_FAILURE(status)) {
             log_err_status(status, "Error getting keyword values: %s\n", u_errorName(status));
             break;
         }
         size = uenum_count(keywordValues, &status);
 
-        if (size == expectedLength[i]) {
-            for (n = 0; n < expectedLength[i]; n++) {
-                if ((value = uenum_next(keywordValues, &valueLength, &status)) != NULL && U_SUCCESS(status)) {
-                    if (uprv_strcmp(value, PREFERRED[i][n+1]) != 0) {
-                        log_err("Keyword values differ: Got [%s] Expected [%s] for locale: %s\n", value, PREFERRED[i][n+1], locale);
-                        errorOccurred = TRUE;
-                        break;
-                    }
-
+        for (n = 0; (value = PREFERRED[i][n+1]) != NULL; n++) {
+            if (!uenum_contains(keywordValues, value, &status)) {
+                if (U_SUCCESS(status)) {
+                    log_err("Keyword value \"%s\" missing for locale: %s\n", value, locale);
                 } else {
                     log_err("While getting keyword value from locale: %s got this error: %s\n", locale, u_errorName(status));
                     errorOccurred = TRUE;
                     break;
                 }
             }
-            if (errorOccurred) {
-                break;
-            }
-        } else {
-            log_err("Number of keywords (%d) does not match expected size (%d) for locale: %s\n", size, expectedLength[i], locale);
-            break;
         }
         uenum_close(keywordValues);
         keywordValues = NULL;
     }
-    if (keywordValues != NULL) {
-        uenum_close(keywordValues);
-    }
+    uenum_close(keywordValues);
 }
 
 static void TestStrcollNull(void) {
     UErrorCode status = U_ZERO_ERROR;
     UCollator *coll;
 
     const UChar u16asc[] = {0x0049, 0x0042, 0x004D, 0};
     const int32_t u16ascLen = 3;
 
     const UChar u16han[] = {0x5c71, 0x5ddd, 0};
     const int32_t u16hanLen = 2;
 
-    const char u8asc[] = {0x49, 0x42, 0x4D, 0};
+    const char *u8asc = "\x49\x42\x4D";
     const int32_t u8ascLen = 3;
 
-    const char u8han[] = {0xE5, 0xB1, 0xB1, 0xE5, 0xB7, 0x9D, 0};
+    const char *u8han = "\xE5\xB1\xB1\xE5\xB7\x9D";
     const int32_t u8hanLen = 6;
 
     coll = ucol_open(NULL, &status);
     if (U_FAILURE(status)) {
         log_err_status(status, "Default Collator creation failed.: %s\n", myErrorName(status));
         return;
     }
 
     /* UChar API */
     if (ucol_strcoll(coll, NULL, 0, NULL, 0) != 0) {
@@ skipping 69 matching lines @@
     }
     status = U_ZERO_ERROR;
     if (ucol_strcollUTF8(coll, NULL, 0, u8han, u8hanLen, &status) >= 0 || U_FAILURE(status)) {
         log_err("ERROR : ucol_strcollUTF8 NULL/0 and u8han/u8hanLen");
     }
 
     ucol_close(coll);
 }
 
 #endif /* #if !UCONFIG_NO_COLLATION */