Update ICU to 54.1 step 1

source/i18n/utf16collationiterator.cpp

Index: source/i18n/utf16collationiterator.cpp
diff --git a/source/i18n/utf16collationiterator.cpp b/source/i18n/utf16collationiterator.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..559ea91ea88492145fbdbb799916af11a743148d
--- /dev/null
+++ b/source/i18n/utf16collationiterator.cpp
@@ -0,0 +1,492 @@
+/*
+*******************************************************************************
+* Copyright (C) 2010-2014, International Business Machines
+* Corporation and others.  All Rights Reserved.
+*******************************************************************************
+* utf16collationiterator.cpp
+*
+* created on: 2010oct27
+* created by: Markus W. Scherer
+*/
+
+#include "unicode/utypes.h"
+
+#if !UCONFIG_NO_COLLATION
+
+#include "charstr.h"
+#include "cmemory.h"
+#include "collation.h"
+#include "collationdata.h"
+#include "collationfcd.h"
+#include "collationiterator.h"
+#include "normalizer2impl.h"
+#include "uassert.h"
+#include "utf16collationiterator.h"
+
+U_NAMESPACE_BEGIN
+
+UTF16CollationIterator::UTF16CollationIterator(const UTF16CollationIterator &other,
+                                               const UChar *newText)
+        : CollationIterator(other),
+          start(newText),
+          pos(newText + (other.pos - other.start)),
+          limit(other.limit == NULL ? NULL : newText + (other.limit - other.start)) {
+}
+
+UTF16CollationIterator::~UTF16CollationIterator() {}
+
+UBool
+UTF16CollationIterator::operator==(const CollationIterator &other) const {
+    if(!CollationIterator::operator==(other)) { return FALSE; }
+    const UTF16CollationIterator &o = static_cast<const UTF16CollationIterator &>(other);
+    // Compare the iterator state but not the text: Assume that the caller does that.
+    return (pos - start) == (o.pos - o.start);
+}
+
+void
+UTF16CollationIterator::resetToOffset(int32_t newOffset) {
+    reset();
+    pos = start + newOffset;
+}
+
+int32_t
+UTF16CollationIterator::getOffset() const {
+    return (int32_t)(pos - start);
+}
+
+uint32_t
+UTF16CollationIterator::handleNextCE32(UChar32 &c, UErrorCode & /*errorCode*/) {
+    if(pos == limit) {
+        c = U_SENTINEL;
+        return Collation::FALLBACK_CE32;
+    }
+    c = *pos++;
+    return UTRIE2_GET32_FROM_U16_SINGLE_LEAD(trie, c);
+}
+
+UChar
+UTF16CollationIterator::handleGetTrailSurrogate() {
+    if(pos == limit) { return 0; }
+    UChar trail;
+    if(U16_IS_TRAIL(trail = *pos)) { ++pos; }
+    return trail;
+}
+
+UBool
+UTF16CollationIterator::foundNULTerminator() {
+    if(limit == NULL) {
+        limit = --pos;
+        return TRUE;
+    } else {
+        return FALSE;
+    }
+}
+
+UChar32
+UTF16CollationIterator::nextCodePoint(UErrorCode & /*errorCode*/) {
+    if(pos == limit) {
+        return U_SENTINEL;
+    }
+    UChar32 c = *pos;
+    if(c == 0 && limit == NULL) {
+        limit = pos;
+        return U_SENTINEL;
+    }
+    ++pos;
+    UChar trail;
+    if(U16_IS_LEAD(c) && pos != limit && U16_IS_TRAIL(trail = *pos)) {
+        ++pos;
+        return U16_GET_SUPPLEMENTARY(c, trail);
+    } else {
+        return c;
+    }
+}
+
+UChar32
+UTF16CollationIterator::previousCodePoint(UErrorCode & /*errorCode*/) {
+    if(pos == start) {
+        return U_SENTINEL;
+    }
+    UChar32 c = *--pos;
+    UChar lead;
+    if(U16_IS_TRAIL(c) && pos != start && U16_IS_LEAD(lead = *(pos - 1))) {
+        --pos;
+        return U16_GET_SUPPLEMENTARY(lead, c);
+    } else {
+        return c;
+    }
+}
+
+void
+UTF16CollationIterator::forwardNumCodePoints(int32_t num, UErrorCode & /*errorCode*/) {
+    while(num > 0 && pos != limit) {
+        UChar32 c = *pos;
+        if(c == 0 && limit == NULL) {
+            limit = pos;
+            break;
+        }
+        ++pos;
+        --num;
+        if(U16_IS_LEAD(c) && pos != limit && U16_IS_TRAIL(*pos)) {
+            ++pos;
+        }
+    }
+}
+
+void
+UTF16CollationIterator::backwardNumCodePoints(int32_t num, UErrorCode & /*errorCode*/) {
+    while(num > 0 && pos != start) {
+        UChar32 c = *--pos;
+        --num;
+        if(U16_IS_TRAIL(c) && pos != start && U16_IS_LEAD(*(pos-1))) {
+            --pos;
+        }
+    }
+}
+
+// FCDUTF16CollationIterator ----------------------------------------------- ***
+
+FCDUTF16CollationIterator::FCDUTF16CollationIterator(const FCDUTF16CollationIterator &other,
+                                                     const UChar *newText)
+        : UTF16CollationIterator(other),
+          rawStart(newText),
+          segmentStart(newText + (other.segmentStart - other.rawStart)),
+          segmentLimit(other.segmentLimit == NULL ? NULL : newText + (other.segmentLimit - other.rawStart)),
+          rawLimit(other.rawLimit == NULL ? NULL : newText + (other.rawLimit - other.rawStart)),
+          nfcImpl(other.nfcImpl),
+          normalized(other.normalized),
+          checkDir(other.checkDir) {
+    if(checkDir != 0 || other.start == other.segmentStart) {
+        start = newText + (other.start - other.rawStart);
+        pos = newText + (other.pos - other.rawStart);
+        limit = other.limit == NULL ? NULL : newText + (other.limit - other.rawStart);
+    } else {
+        start = normalized.getBuffer();
+        pos = start + (other.pos - other.start);
+        limit = start + normalized.length();
+    }
+}
+
+FCDUTF16CollationIterator::~FCDUTF16CollationIterator() {}
+
+UBool
+FCDUTF16CollationIterator::operator==(const CollationIterator &other) const {
+    // Skip the UTF16CollationIterator and call its parent.
+    if(!CollationIterator::operator==(other)) { return FALSE; }
+    const FCDUTF16CollationIterator &o = static_cast<const FCDUTF16CollationIterator &>(other);
+    // Compare the iterator state but not the text: Assume that the caller does that.
+    if(checkDir != o.checkDir) { return FALSE; }
+    if(checkDir == 0 && (start == segmentStart) != (o.start == o.segmentStart)) { return FALSE; }
+    if(checkDir != 0 || start == segmentStart) {
+        return (pos - rawStart) == (o.pos - o.rawStart);
+    } else {
+        return (segmentStart - rawStart) == (o.segmentStart - o.rawStart) &&
+                (pos - start) == (o.pos - o.start);
+    }
+}
+
+void
+FCDUTF16CollationIterator::resetToOffset(int32_t newOffset) {
+    reset();
+    start = segmentStart = pos = rawStart + newOffset;
+    limit = rawLimit;
+    checkDir = 1;
+}
+
+int32_t
+FCDUTF16CollationIterator::getOffset() const {
+    if(checkDir != 0 || start == segmentStart) {
+        return (int32_t)(pos - rawStart);
+    } else if(pos == start) {
+        return (int32_t)(segmentStart - rawStart);
+    } else {
+        return (int32_t)(segmentLimit - rawStart);
+    }
+}
+
+uint32_t
+FCDUTF16CollationIterator::handleNextCE32(UChar32 &c, UErrorCode &errorCode) {
+    for(;;) {
+        if(checkDir > 0) {
+            if(pos == limit) {
+                c = U_SENTINEL;
+                return Collation::FALLBACK_CE32;
+            }
+            c = *pos++;
+            if(CollationFCD::hasTccc(c)) {
+                if(CollationFCD::maybeTibetanCompositeVowel(c) ||
+                        (pos != limit && CollationFCD::hasLccc(*pos))) {
+                    --pos;
+                    if(!nextSegment(errorCode)) {
+                        c = U_SENTINEL;
+                        return Collation::FALLBACK_CE32;
+                    }
+                    c = *pos++;
+                }
+            }
+            break;
+        } else if(checkDir == 0 && pos != limit) {
+            c = *pos++;
+            break;
+        } else {
+            switchToForward();
+        }
+    }
+    return UTRIE2_GET32_FROM_U16_SINGLE_LEAD(trie, c);
+}
+
+UBool
+FCDUTF16CollationIterator::foundNULTerminator() {
+    if(limit == NULL) {
+        limit = rawLimit = --pos;
+        return TRUE;
+    } else {
+        return FALSE;
+    }
+}
+
+UChar32
+FCDUTF16CollationIterator::nextCodePoint(UErrorCode &errorCode) {
+    UChar32 c;
+    for(;;) {
+        if(checkDir > 0) {
+            if(pos == limit) {
+                return U_SENTINEL;
+            }
+            c = *pos++;
+            if(CollationFCD::hasTccc(c)) {
+                if(CollationFCD::maybeTibetanCompositeVowel(c) ||
+                        (pos != limit && CollationFCD::hasLccc(*pos))) {
+                    --pos;
+                    if(!nextSegment(errorCode)) {
+                        return U_SENTINEL;
+                    }
+                    c = *pos++;
+                }
+            } else if(c == 0 && limit == NULL) {
+                limit = rawLimit = --pos;
+                return U_SENTINEL;
+            }
+            break;
+        } else if(checkDir == 0 && pos != limit) {
+            c = *pos++;
+            break;
+        } else {
+            switchToForward();
+        }
+    }
+    UChar trail;
+    if(U16_IS_LEAD(c) && pos != limit && U16_IS_TRAIL(trail = *pos)) {
+        ++pos;
+        return U16_GET_SUPPLEMENTARY(c, trail);
+    } else {
+        return c;
+    }
+}
+
+UChar32
+FCDUTF16CollationIterator::previousCodePoint(UErrorCode &errorCode) {
+    UChar32 c;
+    for(;;) {
+        if(checkDir < 0) {
+            if(pos == start) {
+                return U_SENTINEL;
+            }
+            c = *--pos;
+            if(CollationFCD::hasLccc(c)) {
+                if(CollationFCD::maybeTibetanCompositeVowel(c) ||
+                        (pos != start && CollationFCD::hasTccc(*(pos - 1)))) {
+                    ++pos;
+                    if(!previousSegment(errorCode)) {
+                        return U_SENTINEL;
+                    }
+                    c = *--pos;
+                }
+            }
+            break;
+        } else if(checkDir == 0 && pos != start) {
+            c = *--pos;
+            break;
+        } else {
+            switchToBackward();
+        }
+    }
+    UChar lead;
+    if(U16_IS_TRAIL(c) && pos != start && U16_IS_LEAD(lead = *(pos - 1))) {
+        --pos;
+        return U16_GET_SUPPLEMENTARY(lead, c);
+    } else {
+        return c;
+    }
+}
+
+void
+FCDUTF16CollationIterator::forwardNumCodePoints(int32_t num, UErrorCode &errorCode) {
+    // Specify the class to avoid a virtual-function indirection.
+    // In Java, we would declare this class final.
+    while(num > 0 && FCDUTF16CollationIterator::nextCodePoint(errorCode) >= 0) {
+        --num;
+    }
+}
+
+void
+FCDUTF16CollationIterator::backwardNumCodePoints(int32_t num, UErrorCode &errorCode) {
+    // Specify the class to avoid a virtual-function indirection.
+    // In Java, we would declare this class final.
+    while(num > 0 && FCDUTF16CollationIterator::previousCodePoint(errorCode) >= 0) {
+        --num;
+    }
+}
+
+void
+FCDUTF16CollationIterator::switchToForward() {
+    U_ASSERT(checkDir < 0 || (checkDir == 0 && pos == limit));
+    if(checkDir < 0) {
+        // Turn around from backward checking.
+        start = segmentStart = pos;
+        if(pos == segmentLimit) {
+            limit = rawLimit;
+            checkDir = 1;  // Check forward.
+        } else {  // pos < segmentLimit
+            checkDir = 0;  // Stay in FCD segment.
+        }
+    } else {
+        // Reached the end of the FCD segment.
+        if(start == segmentStart) {
+            // The input text segment is FCD, extend it forward.
+        } else {
+            // The input text segment needed to be normalized.
+            // Switch to checking forward from it.
+            pos = start = segmentStart = segmentLimit;
+            // Note: If this segment is at the end of the input text,
+            // then it might help to return FALSE to indicate that, so that
+            // we do not have to re-check and normalize when we turn around and go backwards.
+            // However, that would complicate the call sites for an optimization of an unusual case.
+        }
+        limit = rawLimit;
+        checkDir = 1;
+    }
+}
+
+UBool
+FCDUTF16CollationIterator::nextSegment(UErrorCode &errorCode) {
+    if(U_FAILURE(errorCode)) { return FALSE; }
+    U_ASSERT(checkDir > 0 && pos != limit);
+    // The input text [segmentStart..pos[ passes the FCD check.
+    const UChar *p = pos;
+    uint8_t prevCC = 0;
+    for(;;) {
+        // Fetch the next character's fcd16 value.
+        const UChar *q = p;
+        uint16_t fcd16 = nfcImpl.nextFCD16(p, rawLimit);
+        uint8_t leadCC = (uint8_t)(fcd16 >> 8);
+        if(leadCC == 0 && q != pos) {
+            // FCD boundary before the [q, p[ character.
+            limit = segmentLimit = q;
+            break;
+        }
+        if(leadCC != 0 && (prevCC > leadCC || CollationFCD::isFCD16OfTibetanCompositeVowel(fcd16))) {
+            // Fails FCD check. Find the next FCD boundary and normalize.
+            do {
+                q = p;
+            } while(p != rawLimit && nfcImpl.nextFCD16(p, rawLimit) > 0xff);
+            if(!normalize(pos, q, errorCode)) { return FALSE; }
+            pos = start;
+            break;
+        }
+        prevCC = (uint8_t)fcd16;
+        if(p == rawLimit || prevCC == 0) {
+            // FCD boundary after the last character.
+            limit = segmentLimit = p;
+            break;
+        }
+    }
+    U_ASSERT(pos != limit);
+    checkDir = 0;
+    return TRUE;
+}
+
+void
+FCDUTF16CollationIterator::switchToBackward() {
+    U_ASSERT(checkDir > 0 || (checkDir == 0 && pos == start));
+    if(checkDir > 0) {
+        // Turn around from forward checking.
+        limit = segmentLimit = pos;
+        if(pos == segmentStart) {
+            start = rawStart;
+            checkDir = -1;  // Check backward.
+        } else {  // pos > segmentStart
+            checkDir = 0;  // Stay in FCD segment.
+        }
+    } else {
+        // Reached the start of the FCD segment.
+        if(start == segmentStart) {
+            // The input text segment is FCD, extend it backward.
+        } else {
+            // The input text segment needed to be normalized.
+            // Switch to checking backward from it.
+            pos = limit = segmentLimit = segmentStart;
+        }
+        start = rawStart;
+        checkDir = -1;
+    }
+}
+
+UBool
+FCDUTF16CollationIterator::previousSegment(UErrorCode &errorCode) {
+    if(U_FAILURE(errorCode)) { return FALSE; }
+    U_ASSERT(checkDir < 0 && pos != start);
+    // The input text [pos..segmentLimit[ passes the FCD check.
+    const UChar *p = pos;
+    uint8_t nextCC = 0;
+    for(;;) {
+        // Fetch the previous character's fcd16 value.
+        const UChar *q = p;
+        uint16_t fcd16 = nfcImpl.previousFCD16(rawStart, p);
+        uint8_t trailCC = (uint8_t)fcd16;
+        if(trailCC == 0 && q != pos) {
+            // FCD boundary after the [p, q[ character.
+            start = segmentStart = q;
+            break;
+        }
+        if(trailCC != 0 && ((nextCC != 0 && trailCC > nextCC) ||
+                            CollationFCD::isFCD16OfTibetanCompositeVowel(fcd16))) {
+            // Fails FCD check. Find the previous FCD boundary and normalize.
+            do {
+                q = p;
+            } while(fcd16 > 0xff && p != rawStart &&
+                    (fcd16 = nfcImpl.previousFCD16(rawStart, p)) != 0);
+            if(!normalize(q, pos, errorCode)) { return FALSE; }
+            pos = limit;
+            break;
+        }
+        nextCC = (uint8_t)(fcd16 >> 8);
+        if(p == rawStart || nextCC == 0) {
+            // FCD boundary before the following character.
+            start = segmentStart = p;
+            break;
+        }
+    }
+    U_ASSERT(pos != start);
+    checkDir = 0;
+    return TRUE;
+}
+
+UBool
+FCDUTF16CollationIterator::normalize(const UChar *from, const UChar *to, UErrorCode &errorCode) {
+    // NFD without argument checking.
+    U_ASSERT(U_SUCCESS(errorCode));
+    nfcImpl.decompose(from, to, normalized, (int32_t)(to - from), errorCode);
+    if(U_FAILURE(errorCode)) { return FALSE; }
+    // Switch collation processing into the FCD buffer
+    // with the result of normalizing [segmentStart, segmentLimit[.
+    segmentStart = from;
+    segmentLimit = to;
+    start = normalized.getBuffer();
+    limit = start + normalized.length();
+    return TRUE;
+}
+
+U_NAMESPACE_END
+
+#endif  // !UCONFIG_NO_COLLATION