Index: source/i18n/utf16collationiterator.cpp |
diff --git a/source/i18n/utf16collationiterator.cpp b/source/i18n/utf16collationiterator.cpp |
new file mode 100644 |
index 0000000000000000000000000000000000000000..559ea91ea88492145fbdbb799916af11a743148d |
--- /dev/null |
+++ b/source/i18n/utf16collationiterator.cpp |
@@ -0,0 +1,492 @@ |
+/* |
+******************************************************************************* |
+* Copyright (C) 2010-2014, International Business Machines |
+* Corporation and others. All Rights Reserved. |
+******************************************************************************* |
+* utf16collationiterator.cpp |
+* |
+* created on: 2010oct27 |
+* created by: Markus W. Scherer |
+*/ |
+ |
+#include "unicode/utypes.h" |
+ |
+#if !UCONFIG_NO_COLLATION |
+ |
+#include "charstr.h" |
+#include "cmemory.h" |
+#include "collation.h" |
+#include "collationdata.h" |
+#include "collationfcd.h" |
+#include "collationiterator.h" |
+#include "normalizer2impl.h" |
+#include "uassert.h" |
+#include "utf16collationiterator.h" |
+ |
+U_NAMESPACE_BEGIN |
+ |
+UTF16CollationIterator::UTF16CollationIterator(const UTF16CollationIterator &other, |
+ const UChar *newText) |
+ : CollationIterator(other), |
+ start(newText), |
+ pos(newText + (other.pos - other.start)), |
+ limit(other.limit == NULL ? NULL : newText + (other.limit - other.start)) { |
+} |
+ |
+UTF16CollationIterator::~UTF16CollationIterator() {} |
+ |
+UBool |
+UTF16CollationIterator::operator==(const CollationIterator &other) const { |
+ if(!CollationIterator::operator==(other)) { return FALSE; } |
+ const UTF16CollationIterator &o = static_cast<const UTF16CollationIterator &>(other); |
+ // Compare the iterator state but not the text: Assume that the caller does that. |
+ return (pos - start) == (o.pos - o.start); |
+} |
+ |
+void |
+UTF16CollationIterator::resetToOffset(int32_t newOffset) { |
+ reset(); |
+ pos = start + newOffset; |
+} |
+ |
+int32_t |
+UTF16CollationIterator::getOffset() const { |
+ return (int32_t)(pos - start); |
+} |
+ |
+uint32_t |
+UTF16CollationIterator::handleNextCE32(UChar32 &c, UErrorCode & /*errorCode*/) { |
+ if(pos == limit) { |
+ c = U_SENTINEL; |
+ return Collation::FALLBACK_CE32; |
+ } |
+ c = *pos++; |
+ return UTRIE2_GET32_FROM_U16_SINGLE_LEAD(trie, c); |
+} |
+ |
+UChar |
+UTF16CollationIterator::handleGetTrailSurrogate() { |
+ if(pos == limit) { return 0; } |
+ UChar trail; |
+ if(U16_IS_TRAIL(trail = *pos)) { ++pos; } |
+ return trail; |
+} |
+ |
+UBool |
+UTF16CollationIterator::foundNULTerminator() { |
+ if(limit == NULL) { |
+ limit = --pos; |
+ return TRUE; |
+ } else { |
+ return FALSE; |
+ } |
+} |
+ |
+UChar32 |
+UTF16CollationIterator::nextCodePoint(UErrorCode & /*errorCode*/) { |
+ if(pos == limit) { |
+ return U_SENTINEL; |
+ } |
+ UChar32 c = *pos; |
+ if(c == 0 && limit == NULL) { |
+ limit = pos; |
+ return U_SENTINEL; |
+ } |
+ ++pos; |
+ UChar trail; |
+ if(U16_IS_LEAD(c) && pos != limit && U16_IS_TRAIL(trail = *pos)) { |
+ ++pos; |
+ return U16_GET_SUPPLEMENTARY(c, trail); |
+ } else { |
+ return c; |
+ } |
+} |
+ |
+UChar32 |
+UTF16CollationIterator::previousCodePoint(UErrorCode & /*errorCode*/) { |
+ if(pos == start) { |
+ return U_SENTINEL; |
+ } |
+ UChar32 c = *--pos; |
+ UChar lead; |
+ if(U16_IS_TRAIL(c) && pos != start && U16_IS_LEAD(lead = *(pos - 1))) { |
+ --pos; |
+ return U16_GET_SUPPLEMENTARY(lead, c); |
+ } else { |
+ return c; |
+ } |
+} |
+ |
+void |
+UTF16CollationIterator::forwardNumCodePoints(int32_t num, UErrorCode & /*errorCode*/) { |
+ while(num > 0 && pos != limit) { |
+ UChar32 c = *pos; |
+ if(c == 0 && limit == NULL) { |
+ limit = pos; |
+ break; |
+ } |
+ ++pos; |
+ --num; |
+ if(U16_IS_LEAD(c) && pos != limit && U16_IS_TRAIL(*pos)) { |
+ ++pos; |
+ } |
+ } |
+} |
+ |
+void |
+UTF16CollationIterator::backwardNumCodePoints(int32_t num, UErrorCode & /*errorCode*/) { |
+ while(num > 0 && pos != start) { |
+ UChar32 c = *--pos; |
+ --num; |
+ if(U16_IS_TRAIL(c) && pos != start && U16_IS_LEAD(*(pos-1))) { |
+ --pos; |
+ } |
+ } |
+} |
+ |
+// FCDUTF16CollationIterator ----------------------------------------------- *** |
+ |
+FCDUTF16CollationIterator::FCDUTF16CollationIterator(const FCDUTF16CollationIterator &other, |
+ const UChar *newText) |
+ : UTF16CollationIterator(other), |
+ rawStart(newText), |
+ segmentStart(newText + (other.segmentStart - other.rawStart)), |
+ segmentLimit(other.segmentLimit == NULL ? NULL : newText + (other.segmentLimit - other.rawStart)), |
+ rawLimit(other.rawLimit == NULL ? NULL : newText + (other.rawLimit - other.rawStart)), |
+ nfcImpl(other.nfcImpl), |
+ normalized(other.normalized), |
+ checkDir(other.checkDir) { |
+ if(checkDir != 0 || other.start == other.segmentStart) { |
+ start = newText + (other.start - other.rawStart); |
+ pos = newText + (other.pos - other.rawStart); |
+ limit = other.limit == NULL ? NULL : newText + (other.limit - other.rawStart); |
+ } else { |
+ start = normalized.getBuffer(); |
+ pos = start + (other.pos - other.start); |
+ limit = start + normalized.length(); |
+ } |
+} |
+ |
+FCDUTF16CollationIterator::~FCDUTF16CollationIterator() {} |
+ |
+UBool |
+FCDUTF16CollationIterator::operator==(const CollationIterator &other) const { |
+ // Skip the UTF16CollationIterator and call its parent. |
+ if(!CollationIterator::operator==(other)) { return FALSE; } |
+ const FCDUTF16CollationIterator &o = static_cast<const FCDUTF16CollationIterator &>(other); |
+ // Compare the iterator state but not the text: Assume that the caller does that. |
+ if(checkDir != o.checkDir) { return FALSE; } |
+ if(checkDir == 0 && (start == segmentStart) != (o.start == o.segmentStart)) { return FALSE; } |
+ if(checkDir != 0 || start == segmentStart) { |
+ return (pos - rawStart) == (o.pos - o.rawStart); |
+ } else { |
+ return (segmentStart - rawStart) == (o.segmentStart - o.rawStart) && |
+ (pos - start) == (o.pos - o.start); |
+ } |
+} |
+ |
+void |
+FCDUTF16CollationIterator::resetToOffset(int32_t newOffset) { |
+ reset(); |
+ start = segmentStart = pos = rawStart + newOffset; |
+ limit = rawLimit; |
+ checkDir = 1; |
+} |
+ |
+int32_t |
+FCDUTF16CollationIterator::getOffset() const { |
+ if(checkDir != 0 || start == segmentStart) { |
+ return (int32_t)(pos - rawStart); |
+ } else if(pos == start) { |
+ return (int32_t)(segmentStart - rawStart); |
+ } else { |
+ return (int32_t)(segmentLimit - rawStart); |
+ } |
+} |
+ |
+uint32_t |
+FCDUTF16CollationIterator::handleNextCE32(UChar32 &c, UErrorCode &errorCode) { |
+ for(;;) { |
+ if(checkDir > 0) { |
+ if(pos == limit) { |
+ c = U_SENTINEL; |
+ return Collation::FALLBACK_CE32; |
+ } |
+ c = *pos++; |
+ if(CollationFCD::hasTccc(c)) { |
+ if(CollationFCD::maybeTibetanCompositeVowel(c) || |
+ (pos != limit && CollationFCD::hasLccc(*pos))) { |
+ --pos; |
+ if(!nextSegment(errorCode)) { |
+ c = U_SENTINEL; |
+ return Collation::FALLBACK_CE32; |
+ } |
+ c = *pos++; |
+ } |
+ } |
+ break; |
+ } else if(checkDir == 0 && pos != limit) { |
+ c = *pos++; |
+ break; |
+ } else { |
+ switchToForward(); |
+ } |
+ } |
+ return UTRIE2_GET32_FROM_U16_SINGLE_LEAD(trie, c); |
+} |
+ |
+UBool |
+FCDUTF16CollationIterator::foundNULTerminator() { |
+ if(limit == NULL) { |
+ limit = rawLimit = --pos; |
+ return TRUE; |
+ } else { |
+ return FALSE; |
+ } |
+} |
+ |
+UChar32 |
+FCDUTF16CollationIterator::nextCodePoint(UErrorCode &errorCode) { |
+ UChar32 c; |
+ for(;;) { |
+ if(checkDir > 0) { |
+ if(pos == limit) { |
+ return U_SENTINEL; |
+ } |
+ c = *pos++; |
+ if(CollationFCD::hasTccc(c)) { |
+ if(CollationFCD::maybeTibetanCompositeVowel(c) || |
+ (pos != limit && CollationFCD::hasLccc(*pos))) { |
+ --pos; |
+ if(!nextSegment(errorCode)) { |
+ return U_SENTINEL; |
+ } |
+ c = *pos++; |
+ } |
+ } else if(c == 0 && limit == NULL) { |
+ limit = rawLimit = --pos; |
+ return U_SENTINEL; |
+ } |
+ break; |
+ } else if(checkDir == 0 && pos != limit) { |
+ c = *pos++; |
+ break; |
+ } else { |
+ switchToForward(); |
+ } |
+ } |
+ UChar trail; |
+ if(U16_IS_LEAD(c) && pos != limit && U16_IS_TRAIL(trail = *pos)) { |
+ ++pos; |
+ return U16_GET_SUPPLEMENTARY(c, trail); |
+ } else { |
+ return c; |
+ } |
+} |
+ |
+UChar32 |
+FCDUTF16CollationIterator::previousCodePoint(UErrorCode &errorCode) { |
+ UChar32 c; |
+ for(;;) { |
+ if(checkDir < 0) { |
+ if(pos == start) { |
+ return U_SENTINEL; |
+ } |
+ c = *--pos; |
+ if(CollationFCD::hasLccc(c)) { |
+ if(CollationFCD::maybeTibetanCompositeVowel(c) || |
+ (pos != start && CollationFCD::hasTccc(*(pos - 1)))) { |
+ ++pos; |
+ if(!previousSegment(errorCode)) { |
+ return U_SENTINEL; |
+ } |
+ c = *--pos; |
+ } |
+ } |
+ break; |
+ } else if(checkDir == 0 && pos != start) { |
+ c = *--pos; |
+ break; |
+ } else { |
+ switchToBackward(); |
+ } |
+ } |
+ UChar lead; |
+ if(U16_IS_TRAIL(c) && pos != start && U16_IS_LEAD(lead = *(pos - 1))) { |
+ --pos; |
+ return U16_GET_SUPPLEMENTARY(lead, c); |
+ } else { |
+ return c; |
+ } |
+} |
+ |
+void |
+FCDUTF16CollationIterator::forwardNumCodePoints(int32_t num, UErrorCode &errorCode) { |
+ // Specify the class to avoid a virtual-function indirection. |
+ // In Java, we would declare this class final. |
+ while(num > 0 && FCDUTF16CollationIterator::nextCodePoint(errorCode) >= 0) { |
+ --num; |
+ } |
+} |
+ |
+void |
+FCDUTF16CollationIterator::backwardNumCodePoints(int32_t num, UErrorCode &errorCode) { |
+ // Specify the class to avoid a virtual-function indirection. |
+ // In Java, we would declare this class final. |
+ while(num > 0 && FCDUTF16CollationIterator::previousCodePoint(errorCode) >= 0) { |
+ --num; |
+ } |
+} |
+ |
+void |
+FCDUTF16CollationIterator::switchToForward() { |
+ U_ASSERT(checkDir < 0 || (checkDir == 0 && pos == limit)); |
+ if(checkDir < 0) { |
+ // Turn around from backward checking. |
+ start = segmentStart = pos; |
+ if(pos == segmentLimit) { |
+ limit = rawLimit; |
+ checkDir = 1; // Check forward. |
+ } else { // pos < segmentLimit |
+ checkDir = 0; // Stay in FCD segment. |
+ } |
+ } else { |
+ // Reached the end of the FCD segment. |
+ if(start == segmentStart) { |
+ // The input text segment is FCD, extend it forward. |
+ } else { |
+ // The input text segment needed to be normalized. |
+ // Switch to checking forward from it. |
+ pos = start = segmentStart = segmentLimit; |
+ // Note: If this segment is at the end of the input text, |
+ // then it might help to return FALSE to indicate that, so that |
+ // we do not have to re-check and normalize when we turn around and go backwards. |
+ // However, that would complicate the call sites for an optimization of an unusual case. |
+ } |
+ limit = rawLimit; |
+ checkDir = 1; |
+ } |
+} |
+ |
+UBool |
+FCDUTF16CollationIterator::nextSegment(UErrorCode &errorCode) { |
+ if(U_FAILURE(errorCode)) { return FALSE; } |
+ U_ASSERT(checkDir > 0 && pos != limit); |
+ // The input text [segmentStart..pos[ passes the FCD check. |
+ const UChar *p = pos; |
+ uint8_t prevCC = 0; |
+ for(;;) { |
+ // Fetch the next character's fcd16 value. |
+ const UChar *q = p; |
+ uint16_t fcd16 = nfcImpl.nextFCD16(p, rawLimit); |
+ uint8_t leadCC = (uint8_t)(fcd16 >> 8); |
+ if(leadCC == 0 && q != pos) { |
+ // FCD boundary before the [q, p[ character. |
+ limit = segmentLimit = q; |
+ break; |
+ } |
+ if(leadCC != 0 && (prevCC > leadCC || CollationFCD::isFCD16OfTibetanCompositeVowel(fcd16))) { |
+ // Fails FCD check. Find the next FCD boundary and normalize. |
+ do { |
+ q = p; |
+ } while(p != rawLimit && nfcImpl.nextFCD16(p, rawLimit) > 0xff); |
+ if(!normalize(pos, q, errorCode)) { return FALSE; } |
+ pos = start; |
+ break; |
+ } |
+ prevCC = (uint8_t)fcd16; |
+ if(p == rawLimit || prevCC == 0) { |
+ // FCD boundary after the last character. |
+ limit = segmentLimit = p; |
+ break; |
+ } |
+ } |
+ U_ASSERT(pos != limit); |
+ checkDir = 0; |
+ return TRUE; |
+} |
+ |
+void |
+FCDUTF16CollationIterator::switchToBackward() { |
+ U_ASSERT(checkDir > 0 || (checkDir == 0 && pos == start)); |
+ if(checkDir > 0) { |
+ // Turn around from forward checking. |
+ limit = segmentLimit = pos; |
+ if(pos == segmentStart) { |
+ start = rawStart; |
+ checkDir = -1; // Check backward. |
+ } else { // pos > segmentStart |
+ checkDir = 0; // Stay in FCD segment. |
+ } |
+ } else { |
+ // Reached the start of the FCD segment. |
+ if(start == segmentStart) { |
+ // The input text segment is FCD, extend it backward. |
+ } else { |
+ // The input text segment needed to be normalized. |
+ // Switch to checking backward from it. |
+ pos = limit = segmentLimit = segmentStart; |
+ } |
+ start = rawStart; |
+ checkDir = -1; |
+ } |
+} |
+ |
+UBool |
+FCDUTF16CollationIterator::previousSegment(UErrorCode &errorCode) { |
+ if(U_FAILURE(errorCode)) { return FALSE; } |
+ U_ASSERT(checkDir < 0 && pos != start); |
+ // The input text [pos..segmentLimit[ passes the FCD check. |
+ const UChar *p = pos; |
+ uint8_t nextCC = 0; |
+ for(;;) { |
+ // Fetch the previous character's fcd16 value. |
+ const UChar *q = p; |
+ uint16_t fcd16 = nfcImpl.previousFCD16(rawStart, p); |
+ uint8_t trailCC = (uint8_t)fcd16; |
+ if(trailCC == 0 && q != pos) { |
+ // FCD boundary after the [p, q[ character. |
+ start = segmentStart = q; |
+ break; |
+ } |
+ if(trailCC != 0 && ((nextCC != 0 && trailCC > nextCC) || |
+ CollationFCD::isFCD16OfTibetanCompositeVowel(fcd16))) { |
+ // Fails FCD check. Find the previous FCD boundary and normalize. |
+ do { |
+ q = p; |
+ } while(fcd16 > 0xff && p != rawStart && |
+ (fcd16 = nfcImpl.previousFCD16(rawStart, p)) != 0); |
+ if(!normalize(q, pos, errorCode)) { return FALSE; } |
+ pos = limit; |
+ break; |
+ } |
+ nextCC = (uint8_t)(fcd16 >> 8); |
+ if(p == rawStart || nextCC == 0) { |
+ // FCD boundary before the following character. |
+ start = segmentStart = p; |
+ break; |
+ } |
+ } |
+ U_ASSERT(pos != start); |
+ checkDir = 0; |
+ return TRUE; |
+} |
+ |
+UBool |
+FCDUTF16CollationIterator::normalize(const UChar *from, const UChar *to, UErrorCode &errorCode) { |
+ // NFD without argument checking. |
+ U_ASSERT(U_SUCCESS(errorCode)); |
+ nfcImpl.decompose(from, to, normalized, (int32_t)(to - from), errorCode); |
+ if(U_FAILURE(errorCode)) { return FALSE; } |
+ // Switch collation processing into the FCD buffer |
+ // with the result of normalizing [segmentStart, segmentLimit[. |
+ segmentStart = from; |
+ segmentLimit = to; |
+ start = normalized.getBuffer(); |
+ limit = start + normalized.length(); |
+ return TRUE; |
+} |
+ |
+U_NAMESPACE_END |
+ |
+#endif // !UCONFIG_NO_COLLATION |