ICU 56 update step 1

source/test/intltest/collationtest.cpp

 /*
 *******************************************************************************
-* Copyright (C) 2012-2014, International Business Machines
+* Copyright (C) 2012-2015, International Business Machines
 * Corporation and others.  All Rights Reserved.
 *******************************************************************************
 * collationtest.cpp
 *
 * created on: 2012apr27
 * created by: Markus W. Scherer
 */
 
 #include "unicode/utypes.h"
 
@@ skipping 27 matching lines @@
 #include "normalizer2impl.h"
 #include "ucbuf.h"
 #include "uhash.h"
 #include "uitercollationiterator.h"
 #include "utf16collationiterator.h"
 #include "utf8collationiterator.h"
 #include "uvectr32.h"
 #include "uvectr64.h"
 #include "writesrc.h"
 
-// TODO: Move to ucbuf.h
-U_DEFINE_LOCAL_OPEN_POINTER(LocalUCHARBUFPointer, UCHARBUF, ucbuf_close);
-
 class CodePointIterator;
 
 // TODO: try to share code with IntlTestCollator; for example, prettify(CollationKey)
 
 class CollationTest : public IntlTest {
 public:
     CollationTest()
             : fcd(NULL), nfd(NULL),
               fileLineNumber(0),
               coll(NULL) {}
@@ skipping 26 matching lines @@
 
     // Helpers & fields for data-driven test.
     static UBool isCROrLF(UChar c) { return c == 0xa || c == 0xd; }
     static UBool isSpace(UChar c) { return c == 9 || c == 0x20 || c == 0x3000; }
     static UBool isSectionStarter(UChar c) { return c == 0x25 || c == 0x2a || c == 0x40; }  // %*@
     int32_t skipSpaces(int32_t i) {
         while(isSpace(fileLine[i])) { ++i; }
         return i;
     }
 
-    UBool readLine(UCHARBUF *f, IcuTestErrorCode &errorCode);
+    UBool readNonEmptyLine(UCHARBUF *f, IcuTestErrorCode &errorCode);
     void parseString(int32_t &start, UnicodeString &prefix, UnicodeString &s, UErrorCode &errorCode);
     Collation::Level parseRelationAndString(UnicodeString &s, IcuTestErrorCode &errorCode);
     void parseAndSetAttribute(IcuTestErrorCode &errorCode);
     void parseAndSetReorderCodes(int32_t start, IcuTestErrorCode &errorCode);
     void buildTailoring(UCHARBUF *f, IcuTestErrorCode &errorCode);
     void setRootCollator(IcuTestErrorCode &errorCode);
     void setLocaleCollator(IcuTestErrorCode &errorCode);
 
     UBool needsNormalization(const UnicodeString &s, UErrorCode &errorCode) const;
 
     UBool getSortKeyParts(const UChar *s, int32_t length,
                           CharString &dest, int32_t partSize,
                           IcuTestErrorCode &errorCode);
     UBool getCollationKey(const char *norm, const UnicodeString &line,
                           const UChar *s, int32_t length,
                           CollationKey &key, IcuTestErrorCode &errorCode);
+    UBool getMergedCollationKey(const UChar *s, int32_t length,
+                                CollationKey &key, IcuTestErrorCode &errorCode);
     UBool checkCompareTwo(const char *norm, const UnicodeString &prevFileLine,
                           const UnicodeString &prevString, const UnicodeString &s,
                           UCollationResult expectedOrder, Collation::Level expectedLevel,
                           IcuTestErrorCode &errorCode);
     void checkCompareStrings(UCHARBUF *f, IcuTestErrorCode &errorCode);
 
     const Normalizer2 *fcd, *nfd;
     UnicodeString fileLine;
     int32_t fileLineNumber;
     UnicodeString fileTestName;
@@ skipping 38 matching lines @@
 
     static const UChar s[2] = { 0xfffe, 0xffff };
     UVector64 ces(errorCode);
     rbc->internalGetCEs(UnicodeString(FALSE, s, 2), ces, errorCode);
     errorCode.assertSuccess();
     if(ces.size() != 2) {
         errln("expected 2 CEs for <FFFE, FFFF>, got %d", (int)ces.size());
         return;
     }
     int64_t ce = ces.elementAti(0);
-    int64_t expected =
-        ((int64_t)Collation::MERGE_SEPARATOR_PRIMARY << 32) |
-        Collation::MERGE_SEPARATOR_LOWER32;
+    int64_t expected = Collation::makeCE(Collation::MERGE_SEPARATOR_PRIMARY);
     if(ce != expected) {
-        errln("CE(U+fffe)=%04lx != 02.02.02", (long)ce);
+        errln("CE(U+fffe)=%04lx != 02..", (long)ce);
     }
 
     ce = ces.elementAti(1);
     expected = Collation::makeCE(Collation::MAX_PRIMARY);
     if(ce != expected) {
         errln("CE(U+ffff)=%04lx != max..", (long)ce);
     }
 }
 
 void CollationTest::TestImplicits() {
     IcuTestErrorCode errorCode(*this, "TestImplicits");
 
     const CollationData *cd = CollationRoot::getData(errorCode);
-    if(errorCode.logDataIfFailureAndReset("CollationRoot::getBaseData()")) {
+    if(errorCode.logDataIfFailureAndReset("CollationRoot::getData()")) {
         return;
     }
 
     // Implicit primary weights should be assigned for the following sets,
     // and sort in ascending order by set and then code point.
     // See http://www.unicode.org/reports/tr10/#Implicit_Weights
 
     // core Han Unified Ideographs
     UnicodeSet coreHan("[\\p{unified_ideograph}&"
                             "[\\p{Block=CJK_Unified_Ideographs}"
@@ skipping 406 matching lines @@
     }
     // No intermediate zero bytes.
     if(p1 != 0 && p2 == 0 && (p & 0xffff) != 0) {
         return FALSE;
     }
     if(p2 != 0 && p3 == 0 && p4 != 0) {
         return FALSE;
     }
     // Minimum & maximum lead bytes.
     if((p1 != 0 && p1 <= Collation::MERGE_SEPARATOR_BYTE) ||
-            (s1 != 0 && s1 <= Collation::MERGE_SEPARATOR_BYTE) ||
-            (t1 != 0 && t1 <= Collation::MERGE_SEPARATOR_BYTE)) {
-        return FALSE;
-    }
-    if(t1 != 0 && t1 > 0x3f) {
+            s1 == Collation::LEVEL_SEPARATOR_BYTE ||
+            t1 == Collation::LEVEL_SEPARATOR_BYTE || t1 > 0x3f) {
         return FALSE;
     }
     if(c > 2) {
         return FALSE;
     }
     // The valid byte range for the second primary byte depends on compressibility.
     if(p2 != 0) {
         if(data.isCompressibleLeadByte(p1)) {
             if(p2 <= Collation::PRIMARY_COMPRESSION_LOW_BYTE ||
                     Collation::PRIMARY_COMPRESSION_HIGH_BYTE <= p2) {
@@ skipping 84 matching lines @@
             if((pri & 0xffff) == 0) {
                 pri = Collation::incTwoBytePrimaryByOffset(pri, isCompressible, step);
             } else {
                 pri = Collation::incThreeBytePrimaryByOffset(pri, isCompressible, step);
             }
             return TRUE;
         }
         // Simple primary CE.
         ++index;
         pri = p;
-        secTer = Collation::COMMON_SEC_AND_TER_CE;
+        // Does this have an explicit below-common sec/ter unit,
+        // or does it imply a common one?
+        if(index == length) {
+            secTer = Collation::COMMON_SEC_AND_TER_CE;
+        } else {
+            secTer = elements[index];
+            if((secTer & CollationRootElements::SEC_TER_DELTA_FLAG) == 0) {
+                // No sec/ter delta.
+                secTer = Collation::COMMON_SEC_AND_TER_CE;
+            } else {
+                secTer &= ~CollationRootElements::SEC_TER_DELTA_FLAG;
+                if(secTer > Collation::COMMON_SEC_AND_TER_CE) {
+                    // Implied sec/ter.
+                    secTer = Collation::COMMON_SEC_AND_TER_CE;
+                } else {
+                    // Explicit sec/ter below common/common.
+                    ++index;
+                }
+            }
+        }
         return TRUE;
     }
 
     uint32_t getPrimary() const { return pri; }
     uint32_t getSecTer() const { return secTer; }
 
 private:
     const CollationData &data;
     const uint32_t *elements;
     int32_t length;
@@ skipping 206 matching lines @@
     }
     return s;
 }
 
 UnicodeString CollationTest::printCollationKey(const CollationKey &key) {
     int32_t length;
     const uint8_t *p = key.getByteArray(length);
     return printSortKey(p, length);
 }
 
-UBool CollationTest::readLine(UCHARBUF *f, IcuTestErrorCode &errorCode) {
-    int32_t lineLength;
-    const UChar *line = ucbuf_readline(f, &lineLength, errorCode);
-    if(line == NULL || errorCode.isFailure()) {
-        fileLine.remove();
-        return FALSE;
+UBool CollationTest::readNonEmptyLine(UCHARBUF *f, IcuTestErrorCode &errorCode) {
+    for(;;) {
+        int32_t lineLength;
+        const UChar *line = ucbuf_readline(f, &lineLength, errorCode);
+        if(line == NULL || errorCode.isFailure()) {
+            fileLine.remove();
+            return FALSE;
+        }
+        ++fileLineNumber;
+        // Strip trailing CR/LF, comments, and spaces.
+        const UChar *comment = u_memchr(line, 0x23, lineLength);  // '#'
+        if(comment != NULL) {
+            lineLength = (int32_t)(comment - line);
+        } else {
+            while(lineLength > 0 && isCROrLF(line[lineLength - 1])) { --lineLength; }
+        }
+        while(lineLength > 0 && isSpace(line[lineLength - 1])) { --lineLength; }
+        if(lineLength != 0) {
+            fileLine.setTo(FALSE, line, lineLength);
+            return TRUE;
+        }
+        // Empty line, continue.
     }
-    ++fileLineNumber;
-    // Strip trailing CR/LF, comments, and spaces.
-    const UChar *comment = u_memchr(line, 0x23, lineLength);  // '#'
-    if(comment != NULL) {
-        lineLength = (int32_t)(comment - line);
-    } else {
-        while(lineLength > 0 && isCROrLF(line[lineLength - 1])) { --lineLength; }
-    }
-    while(lineLength > 0 && isSpace(line[lineLength - 1])) { --lineLength; }
-    fileLine.setTo(FALSE, line, lineLength);
-    return TRUE;
 }
 
 void CollationTest::parseString(int32_t &start, UnicodeString &prefix, UnicodeString &s,
                                 UErrorCode &errorCode) {
     int32_t length = fileLine.length();
     int32_t i;
     for(i = start; i < length && !isSpace(fileLine[i]); ++i) {}
     int32_t pipeIndex = fileLine.indexOf((UChar)0x7c, start, i - start);  // '|'
     if(pipeIndex >= 0) {
         prefix = fileLine.tempSubStringBetween(start, pipeIndex).unescape();
@@ skipping 103 matching lines @@
     { "identical", UCOL_IDENTICAL },
     { "off", UCOL_OFF },
     { "on", UCOL_ON },
     { "shifted", UCOL_SHIFTED },
     { "non-ignorable", UCOL_NON_IGNORABLE },
     { "lower", UCOL_LOWER_FIRST },
     { "upper", UCOL_UPPER_FIRST }
 };
 
 void CollationTest::parseAndSetAttribute(IcuTestErrorCode &errorCode) {
+    // Parse attributes even if the Collator could not be created,
+    // in order to report syntax errors.
     int32_t start = skipSpaces(1);
     int32_t equalPos = fileLine.indexOf(0x3d);
     if(equalPos < 0) {
         if(fileLine.compare(start, 7, UNICODE_STRING("reorder", 7)) == 0) {
             parseAndSetReorderCodes(start + 7, errorCode);
             return;
         }
         errln("missing '=' on line %d", (int)fileLineNumber);
         infoln(fileLine);
         errorCode.set(U_PARSE_ERROR);
@@ skipping 11 matching lines @@
         } else if(valueString == UNICODE_STRING("symbol", 6)) {
             max = UCOL_REORDER_CODE_SYMBOL;
         } else if(valueString == UNICODE_STRING("currency", 8)) {
             max = UCOL_REORDER_CODE_CURRENCY;
         } else {
             errln("invalid attribute value name on line %d", (int)fileLineNumber);
             infoln(fileLine);
             errorCode.set(U_PARSE_ERROR);
             return;
         }
-        coll->setMaxVariable(max, errorCode);
-        if(errorCode.isFailure()) {
-            errln("setMaxVariable() failed on line %d: %s",
-                  (int)fileLineNumber, errorCode.errorName());
-            infoln(fileLine);
-            return;
+        if(coll != NULL) {
+            coll->setMaxVariable(max, errorCode);
+            if(errorCode.isFailure()) {
+                errln("setMaxVariable() failed on line %d: %s",
+                      (int)fileLineNumber, errorCode.errorName());
+                infoln(fileLine);
+                return;
+            }
         }
         fileLine.remove();
         return;
     }
 
     UColAttribute attr;
     for(int32_t i = 0;; ++i) {
         if(i == UPRV_LENGTHOF(attributes)) {
             errln("invalid attribute name on line %d", (int)fileLineNumber);
             infoln(fileLine);
@@ skipping 13 matching lines @@
             infoln(fileLine);
             errorCode.set(U_PARSE_ERROR);
             return;
         }
         if(valueString == UnicodeString(attributeValues[i].name, -1, US_INV)) {
             value = attributeValues[i].value;
             break;
         }
     }
 
-    coll->setAttribute(attr, value, errorCode);
-    if(errorCode.isFailure()) {
-        errln("illegal attribute=value combination on line %d: %s",
-              (int)fileLineNumber, errorCode.errorName());
-        infoln(fileLine);
-        return;
+    if(coll != NULL) {
+        coll->setAttribute(attr, value, errorCode);
+        if(errorCode.isFailure()) {
+            errln("illegal attribute=value combination on line %d: %s",
+                  (int)fileLineNumber, errorCode.errorName());
+            infoln(fileLine);
+            return;
+        }
     }
     fileLine.remove();
 }
 
 void CollationTest::parseAndSetReorderCodes(int32_t start, IcuTestErrorCode &errorCode) {
     UVector32 reorderCodes(errorCode);
     while(start < fileLine.length()) {
         start = skipSpaces(start);
         int32_t limit = start;
         while(limit < fileLine.length() && !isSpace(fileLine[limit])) { ++limit; }
         CharString name;
         name.appendInvariantChars(fileLine.tempSubStringBetween(start, limit), errorCode);
         int32_t code = CollationRuleParser::getReorderCode(name.data());
         if(code < 0) {
             if(uprv_stricmp(name.data(), "default") == 0) {
                 code = UCOL_REORDER_CODE_DEFAULT;  // -1
             } else {
                 errln("invalid reorder code '%s' on line %d", name.data(), (int)fileLineNumber);
                 infoln(fileLine);
                 errorCode.set(U_PARSE_ERROR);
                 return;
             }
         }
         reorderCodes.addElement(code, errorCode);
         start = limit;
     }
-    coll->setReorderCodes(reorderCodes.getBuffer(), reorderCodes.size(), errorCode);
-    if(errorCode.isFailure()) {
-        errln("setReorderCodes() failed on line %d: %s", (int)fileLineNumber, errorCode.errorName());
-        infoln(fileLine);
-        return;
+    if(coll != NULL) {
+        coll->setReorderCodes(reorderCodes.getBuffer(), reorderCodes.size(), errorCode);
+        if(errorCode.isFailure()) {
+            errln("setReorderCodes() failed on line %d: %s",
+                  (int)fileLineNumber, errorCode.errorName());
+            infoln(fileLine);
+            return;
+        }
     }
     fileLine.remove();
 }
 
 void CollationTest::buildTailoring(UCHARBUF *f, IcuTestErrorCode &errorCode) {
     UnicodeString rules;
-    while(readLine(f, errorCode)) {
-        if(fileLine.isEmpty()) { continue; }
-        if(isSectionStarter(fileLine[0])) { break; }
+    while(readNonEmptyLine(f, errorCode) && !isSectionStarter(fileLine[0])) {
         rules.append(fileLine.unescape());
     }
     if(errorCode.isFailure()) { return; }
     logln(rules);
 
     UParseError parseError;
     UnicodeString reason;
     delete coll;
     coll = new RuleBasedCollator(rules, parseError, reason, errorCode);
     if(coll == NULL) {
         errln("unable to allocate a new collator");
         errorCode.set(U_MEMORY_ALLOCATION_ERROR);
         return;
     }
     if(errorCode.isFailure()) {
         dataerrln("RuleBasedCollator(rules) failed - %s", errorCode.errorName());
         infoln(UnicodeString("  reason: ") + reason);
         if(parseError.offset >= 0) { infoln("  rules offset: %d", (int)parseError.offset); }
         if(parseError.preContext[0] != 0 || parseError.postContext[0] != 0) {
             infoln(UnicodeString("  snippet: ...") +
                 parseError.preContext + "(!)" + parseError.postContext + "...");
         }
+        delete coll;
+        coll = NULL;
+        errorCode.reset();
     } else {
         assertEquals("no error reason when RuleBasedCollator(rules) succeeds",
                      UnicodeString(), reason);
     }
 }
 
 void CollationTest::setRootCollator(IcuTestErrorCode &errorCode) {
     if(errorCode.isFailure()) { return; }
     delete coll;
     coll = Collator::createInstance(Locale::getRoot(), errorCode);
     if(errorCode.isFailure()) {
         dataerrln("unable to create a root collator");
         return;
     }
 }
 
 void CollationTest::setLocaleCollator(IcuTestErrorCode &errorCode) {
     if(errorCode.isFailure()) { return; }
+    delete coll;
+    coll = NULL;
     int32_t at = fileLine.indexOf((UChar)0x40, 9);  // @ is not invariant
     if(at >= 0) {
         fileLine.setCharAt(at, (UChar)0x2a);  // *
     }
     CharString localeID;
     localeID.appendInvariantChars(fileLine.tempSubString(9), errorCode);
     if(at >= 0) {
         localeID.data()[at - 9] = '@';
     }
     Locale locale(localeID.data());
     if(fileLine.length() == 9 || errorCode.isFailure() || locale.isBogus()) {
         errln("invalid language tag on line %d", (int)fileLineNumber);
         infoln(fileLine);
         if(errorCode.isSuccess()) { errorCode.set(U_PARSE_ERROR); }
         return;
     }
 
     logln("creating a collator for locale ID %s", locale.getName());
-    Collator *newColl = Collator::createInstance(locale, errorCode);
+    coll = Collator::createInstance(locale, errorCode);
     if(errorCode.isFailure()) {
         dataerrln("unable to create a collator for locale %s on line %d",
                   locale.getName(), (int)fileLineNumber);
         infoln(fileLine);
-        return;
+        delete coll;
+        coll = NULL;
+        errorCode.reset();
     }
-    delete coll;
-    coll = newColl;
 }
 
 UBool CollationTest::needsNormalization(const UnicodeString &s, UErrorCode &errorCode) const {
     if(U_FAILURE(errorCode) || !fcd->isNormalized(s, errorCode)) { return TRUE; }
     // In some sequences with Tibetan composite vowel signs,
     // even if the string passes the FCD check,
     // those composites must be decomposed.
     // Check if s contains 0F71 immediately followed by 0F73 or 0F75 or 0F81.
     int32_t index = 0;
     while((index = s.indexOf((UChar)0xf71, index)) >= 0) {
@@ skipping 75 matching lines @@
     }
     if(numLevelSeparators != (numLevels - 1)) {
         infoln(fileTestName);
         errln("Collator(%s).getCollationKey() has %d level separators for %d levels",
               norm, (int)numLevelSeparators, (int)numLevels);
         infoln(line);
         infoln(printCollationKey(key));
         return FALSE;
     }
 
-    // If s contains U+FFFE, check that merged segments make the same key.
+    // Check that internalNextSortKeyPart() makes the same key, with several part sizes.
+    static const int32_t partSizes[] = { 32, 3, 1 };
+    for(int32_t psi = 0; psi < UPRV_LENGTHOF(partSizes); ++psi) {
+        int32_t partSize = partSizes[psi];
+        CharString parts;
+        if(!getSortKeyParts(s, length, parts, 32, errorCode)) {
+            infoln(fileTestName);
+            errln("Collator(%s).internalNextSortKeyPart(%d) failed: %s",
+                  norm, (int)partSize, errorCode.errorName());
+            infoln(line);
+            return FALSE;
+        }
+        if(keyLength != parts.length() || uprv_memcmp(keyBytes, parts.data(), keyLength) != 0) {
+            infoln(fileTestName);
+            errln("Collator(%s).getCollationKey() != internalNextSortKeyPart(%d)",
+                  norm, (int)partSize);
+            infoln(line);
+            infoln(printCollationKey(key));
+            infoln(printSortKey(reinterpret_cast<uint8_t *>(parts.data()), parts.length()));
+            return FALSE;
+        }
+    }
+    return TRUE;
+}
+
+/**
+ * Changes the key to the merged segments of the U+FFFE-separated substrings of s.
+ * Leaves key unchanged if s does not contain U+FFFE.
+ * @return TRUE if the key was successfully changed
+ */
+UBool CollationTest::getMergedCollationKey(const UChar *s, int32_t length,
+                                           CollationKey &key, IcuTestErrorCode &errorCode) {
+    if(errorCode.isFailure()) { return FALSE; }
     LocalMemory<uint8_t> mergedKey;
     int32_t mergedKeyLength = 0;
     int32_t mergedKeyCapacity = 0;
     int32_t sLength = (length >= 0) ? length : u_strlen(s);
     int32_t segmentStart = 0;
     for(int32_t i = 0;;) {
         if(i == sLength) {
             if(segmentStart == 0) {
                 // s does not contain any U+FFFE.
-                break;
+                return FALSE;
             }
         } else if(s[i] != 0xfffe) {
             ++i;
             continue;
         }
         // Get the sort key for another segment and merge it into mergedKey.
         CollationKey key1(mergedKey.getAlias(), mergedKeyLength);  // copies the bytes
         CollationKey key2;
         coll->getCollationKey(s + segmentStart, i - segmentStart, key2, errorCode);
         int32_t key1Length, key2Length;
@@ skipping 20 matching lines @@
             uprv_memcpy(dest, key2Bytes, key2Length);
             mergedKeyLength = key2Length;
         } else {
             mergedKeyLength =
                 ucol_mergeSortkeys(key1Bytes, key1Length, key2Bytes, key2Length,
                                    dest, mergedKeyCapacity);
         }
         if(i == sLength) { break; }
         segmentStart = ++i;
     }
-    if(segmentStart != 0 &&
-            (mergedKeyLength != keyLength ||
-            uprv_memcmp(mergedKey.getAlias(), keyBytes, keyLength) != 0)) {
-        infoln(fileTestName);
-        errln("Collator(%s).getCollationKey(with U+FFFE) != "
-              "ucol_mergeSortkeys(segments)",
-              norm);
-        infoln(line);
-        infoln(printCollationKey(key));
-        infoln(printSortKey(mergedKey.getAlias(), mergedKeyLength));
-        return FALSE;
-    }
-
-    // Check that internalNextSortKeyPart() makes the same key, with several part sizes.
-    static const int32_t partSizes[] = { 32, 3, 1 };
-    for(int32_t psi = 0; psi < UPRV_LENGTHOF(partSizes); ++psi) {
-        int32_t partSize = partSizes[psi];
-        CharString parts;
-        if(!getSortKeyParts(s, length, parts, 32, errorCode)) {
-            infoln(fileTestName);
-            errln("Collator(%s).internalNextSortKeyPart(%d) failed: %s",
-                  norm, (int)partSize, errorCode.errorName());
-            infoln(line);
-            return FALSE;
-        }
-        if(keyLength != parts.length() || uprv_memcmp(keyBytes, parts.data(), keyLength) != 0) {
-            infoln(fileTestName);
-            errln("Collator(%s).getCollationKey() != internalNextSortKeyPart(%d)",
-                  norm, (int)partSize);
-            infoln(line);
-            infoln(printCollationKey(key));
-            infoln(printSortKey(reinterpret_cast<uint8_t *>(parts.data()), parts.length()));
-            return FALSE;
-        }
-    }
+    key = CollationKey(mergedKey.getAlias(), mergedKeyLength);
     return TRUE;
 }
 
 namespace {
 
 /**
  * Replaces unpaired surrogates with U+FFFD.
  * Returns s if no replacement was made, otherwise buffer.
  */
 const UnicodeString &surrogatesToFFFD(const UnicodeString &s, UnicodeString &buffer) {
@@ skipping 10 matching lines @@
     }
     if(buffer.isEmpty()) {
         return s;
     }
     if(buffer.length() < i) {
         buffer.append(s, buffer.length(), i - buffer.length());
     }
     return buffer;
 }
 
+int32_t getDifferenceLevel(const CollationKey &prevKey, const CollationKey &key,
+                           UCollationResult order, UBool collHasCaseLevel) {
+    if(order == UCOL_EQUAL) {
+        return Collation::NO_LEVEL;
+    }
+    int32_t prevKeyLength;
+    const uint8_t *prevBytes = prevKey.getByteArray(prevKeyLength);
+    int32_t keyLength;
+    const uint8_t *bytes = key.getByteArray(keyLength);
+    int32_t level = Collation::PRIMARY_LEVEL;
+    for(int32_t i = 0;; ++i) {
+        uint8_t b = prevBytes[i];
+        if(b != bytes[i]) { break; }
+        if(b == Collation::LEVEL_SEPARATOR_BYTE) {
+            ++level;
+            if(level == Collation::CASE_LEVEL && !collHasCaseLevel) {
+                ++level;
+            }
+        }
+    }
+    return level;
+}
+
 }
 
 UBool CollationTest::checkCompareTwo(const char *norm, const UnicodeString &prevFileLine,
                                      const UnicodeString &prevString, const UnicodeString &s,
                                      UCollationResult expectedOrder, Collation::Level expectedLevel,
                                      IcuTestErrorCode &errorCode) {
     if(errorCode.isFailure()) { return FALSE; }
 
     // Get the sort keys first, for error debug output.
     CollationKey prevKey;
@@ skipping 141 matching lines @@
     if(order != expectedOrder || errorCode.isFailure()) {
         infoln(fileTestName);
         errln("line %d Collator(%s).getCollationKey(previous, current).compareTo() wrong order: %d != %d (%s)",
               (int)fileLineNumber, norm, order, expectedOrder, errorCode.errorName());
         infoln(prevFileLine);
         infoln(fileLine);
         infoln(printCollationKey(prevKey));
         infoln(printCollationKey(key));
         return FALSE;
     }
+    UBool collHasCaseLevel = coll->getAttribute(UCOL_CASE_LEVEL, errorCode) == UCOL_ON;
+    int32_t level = getDifferenceLevel(prevKey, key, order, collHasCaseLevel);
     if(order != UCOL_EQUAL && expectedLevel != Collation::NO_LEVEL) {
-        int32_t prevKeyLength;
-        const uint8_t *prevBytes = prevKey.getByteArray(prevKeyLength);
-        int32_t keyLength;
-        const uint8_t *bytes = key.getByteArray(keyLength);
-        int32_t level = Collation::PRIMARY_LEVEL;
-        for(int32_t i = 0;; ++i) {
-            uint8_t b = prevBytes[i];
-            if(b != bytes[i]) { break; }
-            if(b == Collation::LEVEL_SEPARATOR_BYTE) {
-                ++level;
-                if(level == Collation::CASE_LEVEL &&
-                        coll->getAttribute(UCOL_CASE_LEVEL, errorCode) == UCOL_OFF) {
-                    ++level;
-                }
-            }
-        }
         if(level != expectedLevel) {
             infoln(fileTestName);
             errln("line %d Collator(%s).getCollationKey(previous, current).compareTo()=%d wrong level: %d != %d",
                   (int)fileLineNumber, norm, order, level, expectedLevel);
             infoln(prevFileLine);
             infoln(fileLine);
             infoln(printCollationKey(prevKey));
             infoln(printCollationKey(key));
             return FALSE;
         }
     }
+
+    // If either string contains U+FFFE, then their sort keys must compare the same as
+    // the merged sort keys of each string's between-FFFE segments.
+    //
+    // It is not required that
+    //   sortkey(str1 + "\uFFFE" + str2) == mergeSortkeys(sortkey(str1), sortkey(str2))
+    // only that those two methods yield the same order.
+    //
+    // Use bit-wise OR so that getMergedCollationKey() is always called for both strings.
+    if((getMergedCollationKey(prevString.getBuffer(), prevString.length(), prevKey, errorCode) |
+                getMergedCollationKey(s.getBuffer(), s.length(), key, errorCode)) ||
+            errorCode.isFailure()) {
+        order = prevKey.compareTo(key, errorCode);
+        if(order != expectedOrder || errorCode.isFailure()) {
+            infoln(fileTestName);
+            errln("line %d ucol_mergeSortkeys(Collator(%s).getCollationKey"
+                "(previous, current segments between U+FFFE)).compareTo() wrong order: %d != %d (%s)",
+                (int)fileLineNumber, norm, order, expectedOrder, errorCode.errorName());
+            infoln(prevFileLine);
+            infoln(fileLine);
+            infoln(printCollationKey(prevKey));
+            infoln(printCollationKey(key));
+            return FALSE;
+        }
+        int32_t mergedLevel = getDifferenceLevel(prevKey, key, order, collHasCaseLevel);
+        if(order != UCOL_EQUAL && expectedLevel != Collation::NO_LEVEL) {
+            if(mergedLevel != level) {
+                infoln(fileTestName);
+                errln("line %d ucol_mergeSortkeys(Collator(%s).getCollationKey"
+                    "(previous, current segments between U+FFFE)).compareTo()=%d wrong level: %d != %d",
+                    (int)fileLineNumber, norm, order, mergedLevel, level);
+                infoln(prevFileLine);
+                infoln(fileLine);
+                infoln(printCollationKey(prevKey));
+                infoln(printCollationKey(key));
+                return FALSE;
+            }
+        }
+    }
     return TRUE;
 }
 
 void CollationTest::checkCompareStrings(UCHARBUF *f, IcuTestErrorCode &errorCode) {
     if(errorCode.isFailure()) { return; }
     UnicodeString prevFileLine = UNICODE_STRING("(none)", 6);
     UnicodeString prevString, s;
     prevString.getTerminatedBuffer();  // Ensure NUL-termination.
-    while(readLine(f, errorCode)) {
-        if(fileLine.isEmpty()) { continue; }
-        if(isSectionStarter(fileLine[0])) { break; }
+    while(readNonEmptyLine(f, errorCode) && !isSectionStarter(fileLine[0])) {
+        // Parse the line even if it will be ignored (when we do not have a Collator)
+        // in order to report syntax issues.
         Collation::Level relation = parseRelationAndString(s, errorCode);
         if(errorCode.isFailure()) {
             errorCode.reset();
             break;
         }
+        if(coll == NULL) {
+            // We were unable to create the Collator but continue with tests.
+            // Ignore test data for this Collator.
+            // The next Collator creation might work.
+            continue;
+        }
         UCollationResult expectedOrder = (relation == Collation::ZERO_LEVEL) ? UCOL_EQUAL : UCOL_LESS;
         Collation::Level expectedLevel = relation;
         s.getTerminatedBuffer();  // Ensure NUL-termination.
         UBool isOk = TRUE;
         if(!needsNormalization(prevString, errorCode) && !needsNormalization(s, errorCode)) {
             coll->setAttribute(UCOL_NORMALIZATION_MODE, UCOL_OFF, errorCode);
             isOk = checkCompareTwo("normalization=on", prevFileLine, prevString, s,
                                    expectedOrder, expectedLevel, errorCode);
         }
         if(isOk) {
@@ skipping 28 matching lines @@
         return;
     }
 
     CharString path(getSourceTestData(errorCode), errorCode);
     path.appendPathPart("collationtest.txt", errorCode);
     const char *codePage = "UTF-8";
     LocalUCHARBUFPointer f(ucbuf_open(path.data(), &codePage, TRUE, FALSE, errorCode));
     if(errorCode.logIfFailureAndReset("ucbuf_open(collationtest.txt)")) {
         return;
     }
-    while(errorCode.isSuccess()) {
-        // Read a new line if necessary.
-        // Sub-parsers leave the first line set that they do not handle.
-        if(fileLine.isEmpty()) {
-            if(!readLine(f.getAlias(), errorCode)) { break; }
-            continue;
-        }
+    // Read a new line if necessary.
+    // Sub-parsers leave the first line set that they do not handle.
+    while(errorCode.isSuccess() && (!fileLine.isEmpty() || readNonEmptyLine(f.getAlias(), errorCode))) {
         if(!isSectionStarter(fileLine[0])) {
             errln("syntax error on line %d", (int)fileLineNumber);
             infoln(fileLine);
             return;
         }
         if(fileLine.startsWith(UNICODE_STRING("** test: ", 9))) {
             fileTestName = fileLine;
             logln(fileLine);
             fileLine.remove();
         } else if(fileLine == UNICODE_STRING("@ root", 6)) {
@@ skipping 10 matching lines @@
             checkCompareStrings(f.getAlias(), errorCode);
         } else {
             errln("syntax error on line %d", (int)fileLineNumber);
             infoln(fileLine);
             return;
         }
     }
 }
 
 #endif  // !UCONFIG_NO_COLLATION