Index: icu52/patches/segmentation.patch |
=================================================================== |
--- icu52/patches/segmentation.patch (revision 261238) |
+++ icu52/patches/segmentation.patch (working copy) |
@@ -1,3587 +0,0 @@ |
---- source/common/brkeng.cpp 2009-11-11 07:47:22.000000000 -0800 |
-+++ source/common/brkeng.cpp 2011-01-21 14:12:45.479922000 -0800 |
-@@ -226,6 +226,30 @@ |
- case USCRIPT_THAI: |
- engine = new ThaiBreakEngine(dict, status); |
- break; |
-+ |
-+ case USCRIPT_HANGUL: |
-+ engine = new CjkBreakEngine(dict, kKorean, status); |
-+ break; |
-+ |
-+ // use same BreakEngine and dictionary for both Chinese and Japanese |
-+ case USCRIPT_HIRAGANA: |
-+ case USCRIPT_KATAKANA: |
-+ case USCRIPT_HAN: |
-+ engine = new CjkBreakEngine(dict, kChineseJapanese, status); |
-+ break; |
-+#if 0 |
-+ // TODO: Have to get some characters with script=common handled |
-+ // by CjkBreakEngine (e.g. U+309B). Simply subjecting |
-+ // them to CjkBreakEngine does not work. The engine has to |
-+ // special-case them. |
-+ case USCRIPT_COMMON: |
-+ { |
-+ UBlockCode block = ublock_getCode(code); |
-+ if (block == UBLOCK_HIRAGANA || block == UBLOCK_KATAKANA) |
-+ engine = new CjkBreakEngine(dict, kChineseJapanese, status); |
-+ break; |
-+ } |
-+#endif |
- default: |
- break; |
- } |
-@@ -281,6 +305,13 @@ |
- dict = NULL; |
- } |
- return dict; |
-+ } else if (dictfname != NULL){ |
-+ //create dummy dict if dictionary filename not valid |
-+ UChar c = 0x0020; |
-+ status = U_ZERO_ERROR; |
-+ MutableTrieDictionary *mtd = new MutableTrieDictionary(c, status, TRUE); |
-+ mtd->addWord(&c, 1, status, 1); |
-+ return new CompactTrieDictionary(*mtd, status); |
- } |
- return NULL; |
- } |
---- source/common/dictbe.cpp 2008-06-13 12:21:12.000000000 -0700 |
-+++ source/common/dictbe.cpp 2011-01-21 14:12:45.468928000 -0800 |
-@@ -16,6 +16,9 @@ |
- #include "unicode/ubrk.h" |
- #include "uvector.h" |
- #include "triedict.h" |
-+#include "uassert.h" |
-+#include "unicode/normlzr.h" |
-+#include "cmemory.h" |
- |
- U_NAMESPACE_BEGIN |
- |
-@@ -422,6 +425,294 @@ |
- return wordsFound; |
- } |
- |
-+/* |
-+ ****************************************************************** |
-+ * CjkBreakEngine |
-+ */ |
-+static const uint32_t kuint32max = 0xFFFFFFFF; |
-+CjkBreakEngine::CjkBreakEngine(const TrieWordDictionary *adoptDictionary, LanguageType type, UErrorCode &status) |
-+: DictionaryBreakEngine(1<<UBRK_WORD), fDictionary(adoptDictionary){ |
-+ if (!adoptDictionary->getValued()) { |
-+ status = U_ILLEGAL_ARGUMENT_ERROR; |
-+ return; |
-+ } |
-+ |
-+ // Korean dictionary only includes Hangul syllables |
-+ fHangulWordSet.applyPattern(UNICODE_STRING_SIMPLE("[\\uac00-\\ud7a3]"), status); |
-+ fHanWordSet.applyPattern(UNICODE_STRING_SIMPLE("[:Han:]"), status); |
-+ fKatakanaWordSet.applyPattern(UNICODE_STRING_SIMPLE("[[:Katakana:]\\uff9e\\uff9f]"), status); |
-+ fHiraganaWordSet.applyPattern(UNICODE_STRING_SIMPLE("[:Hiragana:]"), status); |
-+ |
-+ if (U_SUCCESS(status)) { |
-+ // handle Korean and Japanese/Chinese using different dictionaries |
-+ if (type == kKorean) { |
-+ setCharacters(fHangulWordSet); |
-+ } else { //Chinese and Japanese |
-+ UnicodeSet cjSet; |
-+ cjSet.addAll(fHanWordSet); |
-+ cjSet.addAll(fKatakanaWordSet); |
-+ cjSet.addAll(fHiraganaWordSet); |
-+ cjSet.add(UNICODE_STRING_SIMPLE("\\uff70\\u30fc")); |
-+ setCharacters(cjSet); |
-+ } |
-+ } |
-+} |
-+ |
-+CjkBreakEngine::~CjkBreakEngine(){ |
-+ delete fDictionary; |
-+} |
-+ |
-+// The katakanaCost values below are based on the length frequencies of all |
-+// katakana phrases in the dictionary |
-+static const int kMaxKatakanaLength = 8; |
-+static const int kMaxKatakanaGroupLength = 20; |
-+static const uint32_t maxSnlp = 255; |
-+ |
-+static inline uint32_t getKatakanaCost(int wordLength){ |
-+ //TODO: fill array with actual values from dictionary! |
-+ static const uint32_t katakanaCost[kMaxKatakanaLength + 1] |
-+ = {8192, 984, 408, 240, 204, 252, 300, 372, 480}; |
-+ return (wordLength > kMaxKatakanaLength) ? 8192 : katakanaCost[wordLength]; |
-+} |
-+ |
-+static inline bool isKatakana(uint16_t value) { |
-+ return (value >= 0x30A1u && value <= 0x30FEu && value != 0x30FBu) || |
-+ (value >= 0xFF66u && value <= 0xFF9fu); |
-+} |
-+ |
-+// A very simple helper class to streamline the buffer handling in |
-+// divideUpDictionaryRange. |
-+template<class T, size_t N> |
-+class AutoBuffer { |
-+ public: |
-+ AutoBuffer(size_t size) : buffer(stackBuffer), capacity(N) { |
-+ if (size > N) { |
-+ buffer = reinterpret_cast<T*>(uprv_malloc(sizeof(T)*size)); |
-+ capacity = size; |
-+ } |
-+ } |
-+ ~AutoBuffer() { |
-+ if (buffer != stackBuffer) |
-+ uprv_free(buffer); |
-+ } |
-+#if 0 |
-+ T* operator& () { |
-+ return buffer; |
-+ } |
-+#endif |
-+ T* elems() { |
-+ return buffer; |
-+ } |
-+ const T& operator[] (size_t i) const { |
-+ return buffer[i]; |
-+ } |
-+ T& operator[] (size_t i) { |
-+ return buffer[i]; |
-+ } |
-+ |
-+ // resize without copy |
-+ void resize(size_t size) { |
-+ if (size <= capacity) |
-+ return; |
-+ if (buffer != stackBuffer) |
-+ uprv_free(buffer); |
-+ buffer = reinterpret_cast<T*>(uprv_malloc(sizeof(T)*size)); |
-+ capacity = size; |
-+ } |
-+ private: |
-+ T stackBuffer[N]; |
-+ T* buffer; |
-+ AutoBuffer(); |
-+ size_t capacity; |
-+}; |
-+ |
-+ |
-+/* |
-+ * @param text A UText representing the text |
-+ * @param rangeStart The start of the range of dictionary characters |
-+ * @param rangeEnd The end of the range of dictionary characters |
-+ * @param foundBreaks Output of C array of int32_t break positions, or 0 |
-+ * @return The number of breaks found |
-+ */ |
-+int32_t |
-+CjkBreakEngine::divideUpDictionaryRange( UText *text, |
-+ int32_t rangeStart, |
-+ int32_t rangeEnd, |
-+ UStack &foundBreaks ) const { |
-+ if (rangeStart >= rangeEnd) { |
-+ return 0; |
-+ } |
-+ |
-+ const size_t defaultInputLength = 80; |
-+ size_t inputLength = rangeEnd - rangeStart; |
-+ AutoBuffer<UChar, defaultInputLength> charString(inputLength); |
-+ |
-+ // Normalize the input string and put it in normalizedText. |
-+ // The map from the indices of the normalized input to the raw |
-+ // input is kept in charPositions. |
-+ UErrorCode status = U_ZERO_ERROR; |
-+ utext_extract(text, rangeStart, rangeEnd, charString.elems(), inputLength, &status); |
-+ if (U_FAILURE(status)) |
-+ return 0; |
-+ |
-+ UnicodeString inputString(charString.elems(), inputLength); |
-+ UNormalizationMode norm_mode = UNORM_NFKC; |
-+ UBool isNormalized = |
-+ Normalizer::quickCheck(inputString, norm_mode, status) == UNORM_YES || |
-+ Normalizer::isNormalized(inputString, norm_mode, status); |
-+ |
-+ AutoBuffer<int32_t, defaultInputLength> charPositions(inputLength + 1); |
-+ int numChars = 0; |
-+ UText normalizedText = UTEXT_INITIALIZER; |
-+ // Needs to be declared here because normalizedText holds onto its buffer. |
-+ UnicodeString normalizedString; |
-+ if (isNormalized) { |
-+ int32_t index = 0; |
-+ charPositions[0] = 0; |
-+ while(index < inputString.length()) { |
-+ index = inputString.moveIndex32(index, 1); |
-+ charPositions[++numChars] = index; |
-+ } |
-+ utext_openUnicodeString(&normalizedText, &inputString, &status); |
-+ } |
-+ else { |
-+ Normalizer::normalize(inputString, norm_mode, 0, normalizedString, status); |
-+ if (U_FAILURE(status)) |
-+ return 0; |
-+ charPositions.resize(normalizedString.length() + 1); |
-+ Normalizer normalizer(charString.elems(), inputLength, norm_mode); |
-+ int32_t index = 0; |
-+ charPositions[0] = 0; |
-+ while(index < normalizer.endIndex()){ |
-+ UChar32 uc = normalizer.next(); |
-+ charPositions[++numChars] = index = normalizer.getIndex(); |
-+ } |
-+ utext_openUnicodeString(&normalizedText, &normalizedString, &status); |
-+ } |
-+ |
-+ if (U_FAILURE(status)) |
-+ return 0; |
-+ |
-+ // From this point on, all the indices refer to the indices of |
-+ // the normalized input string. |
-+ |
-+ // bestSnlp[i] is the snlp of the best segmentation of the first i |
-+ // characters in the range to be matched. |
-+ AutoBuffer<uint32_t, defaultInputLength> bestSnlp(numChars + 1); |
-+ bestSnlp[0] = 0; |
-+ for(int i=1; i<=numChars; i++){ |
-+ bestSnlp[i] = kuint32max; |
-+ } |
-+ |
-+ // prev[i] is the index of the last CJK character in the previous word in |
-+ // the best segmentation of the first i characters. |
-+ AutoBuffer<int, defaultInputLength> prev(numChars + 1); |
-+ for(int i=0; i<=numChars; i++){ |
-+ prev[i] = -1; |
-+ } |
-+ |
-+ const size_t maxWordSize = 20; |
-+ AutoBuffer<uint16_t, maxWordSize> values(numChars); |
-+ AutoBuffer<int32_t, maxWordSize> lengths(numChars); |
-+ |
-+ // Dynamic programming to find the best segmentation. |
-+ bool is_prev_katakana = false; |
-+ for (int i = 0; i < numChars; ++i) { |
-+ //utext_setNativeIndex(text, rangeStart + i); |
-+ utext_setNativeIndex(&normalizedText, i); |
-+ if (bestSnlp[i] == kuint32max) |
-+ continue; |
-+ |
-+ int count; |
-+ // limit maximum word length matched to size of current substring |
-+ int maxSearchLength = (i + maxWordSize < (size_t) numChars)? maxWordSize: numChars - i; |
-+ |
-+ fDictionary->matches(&normalizedText, maxSearchLength, lengths.elems(), count, maxSearchLength, values.elems()); |
-+ |
-+ // if there are no single character matches found in the dictionary |
-+ // starting with this charcter, treat character as a 1-character word |
-+ // with the highest value possible, i.e. the least likely to occur. |
-+ // Exclude Korean characters from this treatment, as they should be left |
-+ // together by default. |
-+ if((count == 0 || lengths[0] != 1) && |
-+ !fHangulWordSet.contains(utext_current32(&normalizedText))){ |
-+ values[count] = maxSnlp; |
-+ lengths[count++] = 1; |
-+ } |
-+ |
-+ for (int j = 0; j < count; j++){ |
-+ //U_ASSERT(values[j] >= 0 && values[j] <= maxSnlp); |
-+ uint32_t newSnlp = bestSnlp[i] + values[j]; |
-+ if (newSnlp < bestSnlp[lengths[j] + i]) { |
-+ bestSnlp[lengths[j] + i] = newSnlp; |
-+ prev[lengths[j] + i] = i; |
-+ } |
-+ } |
-+ |
-+ // In Japanese, |
-+ // Katakana word in single character is pretty rare. So we apply |
-+ // the following heuristic to Katakana: any continuous run of Katakana |
-+ // characters is considered a candidate word with a default cost |
-+ // specified in the katakanaCost table according to its length. |
-+ //utext_setNativeIndex(text, rangeStart + i); |
-+ utext_setNativeIndex(&normalizedText, i); |
-+ bool is_katakana = isKatakana(utext_current32(&normalizedText)); |
-+ if (!is_prev_katakana && is_katakana) { |
-+ int j = i + 1; |
-+ utext_next32(&normalizedText); |
-+ // Find the end of the continuous run of Katakana characters |
-+ while (j < numChars && (j - i) < kMaxKatakanaGroupLength && |
-+ isKatakana(utext_current32(&normalizedText))) { |
-+ utext_next32(&normalizedText); |
-+ ++j; |
-+ } |
-+ if ((j - i) < kMaxKatakanaGroupLength) { |
-+ uint32_t newSnlp = bestSnlp[i] + getKatakanaCost(j - i); |
-+ if (newSnlp < bestSnlp[j]) { |
-+ bestSnlp[j] = newSnlp; |
-+ prev[j] = i; |
-+ } |
-+ } |
-+ } |
-+ is_prev_katakana = is_katakana; |
-+ } |
-+ |
-+ // Start pushing the optimal offset index into t_boundary (t for tentative). |
-+ // prev[numChars] is guaranteed to be meaningful. |
-+ // We'll first push in the reverse order, i.e., |
-+ // t_boundary[0] = numChars, and afterwards do a swap. |
-+ AutoBuffer<int, maxWordSize> t_boundary(numChars + 1); |
-+ |
-+ int numBreaks = 0; |
-+ // No segmentation found, set boundary to end of range |
-+ if (bestSnlp[numChars] == kuint32max) { |
-+ t_boundary[numBreaks++] = numChars; |
-+ } else { |
-+ for (int i = numChars; i > 0; i = prev[i]){ |
-+ t_boundary[numBreaks++] = i; |
-+ |
-+ } |
-+ U_ASSERT(prev[t_boundary[numBreaks-1]] == 0); |
-+ } |
-+ |
-+ // Reverse offset index in t_boundary. |
-+ // Don't add a break for the start of the dictionary range if there is one |
-+ // there already. |
-+ if (foundBreaks.size() == 0 || foundBreaks.peeki() < rangeStart) { |
-+ t_boundary[numBreaks++] = 0; |
-+ } |
-+ |
-+ // Now that we're done, convert positions in t_bdry[] (indices in |
-+ // the normalized input string) back to indices in the raw input string |
-+ // while reversing t_bdry and pushing values to foundBreaks. |
-+ for (int i = numBreaks-1; i >= 0; i--) { |
-+ foundBreaks.push(charPositions[t_boundary[i]] + rangeStart, status); |
-+ } |
-+ |
-+ utext_close(&normalizedText); |
-+ return numBreaks; |
-+} |
-+ |
- U_NAMESPACE_END |
- |
- #endif /* #if !UCONFIG_NO_BREAK_ITERATION */ |
---- source/common/dictbe.h 2006-09-29 17:37:45.000000000 -0700 |
-+++ source/common/dictbe.h 2011-01-21 14:12:45.492920000 -0800 |
-@@ -1,8 +1,8 @@ |
- /** |
-- ******************************************************************************* |
-- * Copyright (C) 2006, International Business Machines Corporation and others. * |
-- * All Rights Reserved. * |
-- ******************************************************************************* |
-+ ********************************************************************************** |
-+ * Copyright (C) 2006-2010, International Business Machines Corporation and others. |
-+ * All Rights Reserved. |
-+ ********************************************************************************** |
- */ |
- |
- #ifndef DICTBE_H |
-@@ -65,31 +65,31 @@ |
- */ |
- virtual ~DictionaryBreakEngine(); |
- |
-- /** |
-- * <p>Indicate whether this engine handles a particular character for |
-- * a particular kind of break.</p> |
-- * |
-- * @param c A character which begins a run that the engine might handle |
-- * @param breakType The type of text break which the caller wants to determine |
-- * @return TRUE if this engine handles the particular character and break |
-- * type. |
-- */ |
-+ /** |
-+ * <p>Indicate whether this engine handles a particular character for |
-+ * a particular kind of break.</p> |
-+ * |
-+ * @param c A character which begins a run that the engine might handle |
-+ * @param breakType The type of text break which the caller wants to determine |
-+ * @return TRUE if this engine handles the particular character and break |
-+ * type. |
-+ */ |
- virtual UBool handles( UChar32 c, int32_t breakType ) const; |
- |
-- /** |
-- * <p>Find any breaks within a run in the supplied text.</p> |
-- * |
-- * @param text A UText representing the text. The |
-- * iterator is left at the end of the run of characters which the engine |
-- * is capable of handling. |
-- * @param startPos The start of the run within the supplied text. |
-- * @param endPos The end of the run within the supplied text. |
-- * @param reverse Whether the caller is looking for breaks in a reverse |
-- * direction. |
-- * @param breakType The type of break desired, or -1. |
-- * @param foundBreaks An allocated C array of the breaks found, if any |
-- * @return The number of breaks found. |
-- */ |
-+ /** |
-+ * <p>Find any breaks within a run in the supplied text.</p> |
-+ * |
-+ * @param text A UText representing the text. The iterator is left at |
-+ * the end of the run of characters which the engine is capable of handling |
-+ * that starts from the first (or last) character in the range. |
-+ * @param startPos The start of the run within the supplied text. |
-+ * @param endPos The end of the run within the supplied text. |
-+ * @param reverse Whether the caller is looking for breaks in a reverse |
-+ * direction. |
-+ * @param breakType The type of break desired, or -1. |
-+ * @param foundBreaks An allocated C array of the breaks found, if any |
-+ * @return The number of breaks found. |
-+ */ |
- virtual int32_t findBreaks( UText *text, |
- int32_t startPos, |
- int32_t endPos, |
-@@ -114,7 +114,7 @@ |
- // virtual void setBreakTypes( uint32_t breakTypes ); |
- |
- /** |
-- * <p>Divide up a range of known dictionary characters.</p> |
-+ * <p>Divide up a range of known dictionary characters handled by this break engine.</p> |
- * |
- * @param text A UText representing the text |
- * @param rangeStart The start of the range of dictionary characters |
-@@ -171,7 +171,7 @@ |
- |
- protected: |
- /** |
-- * <p>Divide up a range of known dictionary characters.</p> |
-+ * <p>Divide up a range of known dictionary characters handled by this break engine.</p> |
- * |
- * @param text A UText representing the text |
- * @param rangeStart The start of the range of dictionary characters |
-@@ -186,6 +186,66 @@ |
- |
- }; |
- |
-+/******************************************************************* |
-+ * CjkBreakEngine |
-+ */ |
-+ |
-+//indicates language/script that the CjkBreakEngine will handle |
-+enum LanguageType { |
-+ kKorean, |
-+ kChineseJapanese |
-+}; |
-+ |
-+/** |
-+ * <p>CjkBreakEngine is a kind of DictionaryBreakEngine that uses a |
-+ * TrieWordDictionary with costs associated with each word and |
-+ * Viterbi decoding to determine CJK-specific breaks.</p> |
-+ */ |
-+class CjkBreakEngine : public DictionaryBreakEngine { |
-+ protected: |
-+ /** |
-+ * The set of characters handled by this engine |
-+ * @internal |
-+ */ |
-+ UnicodeSet fHangulWordSet; |
-+ UnicodeSet fHanWordSet; |
-+ UnicodeSet fKatakanaWordSet; |
-+ UnicodeSet fHiraganaWordSet; |
-+ |
-+ const TrieWordDictionary *fDictionary; |
-+ |
-+ public: |
-+ |
-+ /** |
-+ * <p>Default constructor.</p> |
-+ * |
-+ * @param adoptDictionary A TrieWordDictionary to adopt. Deleted when the |
-+ * engine is deleted. The TrieWordDictionary must contain costs for each word |
-+ * in order for the dictionary to work properly. |
-+ */ |
-+ CjkBreakEngine(const TrieWordDictionary *adoptDictionary, LanguageType type, UErrorCode &status); |
-+ |
-+ /** |
-+ * <p>Virtual destructor.</p> |
-+ */ |
-+ virtual ~CjkBreakEngine(); |
-+ |
-+ protected: |
-+ /** |
-+ * <p>Divide up a range of known dictionary characters handled by this break engine.</p> |
-+ * |
-+ * @param text A UText representing the text |
-+ * @param rangeStart The start of the range of dictionary characters |
-+ * @param rangeEnd The end of the range of dictionary characters |
-+ * @param foundBreaks Output of C array of int32_t break positions, or 0 |
-+ * @return The number of breaks found |
-+ */ |
-+ virtual int32_t divideUpDictionaryRange( UText *text, |
-+ int32_t rangeStart, |
-+ int32_t rangeEnd, |
-+ UStack &foundBreaks ) const; |
-+ |
-+}; |
- |
- U_NAMESPACE_END |
- |
---- source/common/rbbi.cpp 2010-07-22 17:15:37.000000000 -0700 |
-+++ source/common/rbbi.cpp 2011-01-21 14:12:45.457938000 -0800 |
-@@ -1555,10 +1555,12 @@ |
- int32_t endPos, |
- UBool reverse) { |
- // Reset the old break cache first. |
-- uint32_t dictionaryCount = fDictionaryCharCount; |
- reset(); |
- |
-- if (dictionaryCount <= 1 || (endPos - startPos) <= 1) { |
-+ // note: code segment below assumes that dictionary chars are in the |
-+ // startPos-endPos range |
-+ // value returned should be next character in sequence |
-+ if ((endPos - startPos) <= 1) { |
- return (reverse ? startPos : endPos); |
- } |
- |
-@@ -1711,7 +1713,7 @@ |
- // proposed break by one of the breaks we found. Use following() and |
- // preceding() to do the work. They should never recurse in this case. |
- if (reverse) { |
-- return preceding(endPos - 1); |
-+ return preceding(endPos); |
- } |
- else { |
- return following(startPos); |
---- source/common/triedict.cpp 2008-02-13 01:35:50.000000000 -0800 |
-+++ source/common/triedict.cpp 2011-01-21 14:12:45.271006000 -0800 |
-@@ -20,6 +20,7 @@ |
- #include "uvector.h" |
- #include "uvectr32.h" |
- #include "uarrsort.h" |
-+#include "hash.h" |
- |
- //#define DEBUG_TRIE_DICT 1 |
- |
-@@ -27,6 +28,11 @@ |
- #include <sys/times.h> |
- #include <limits.h> |
- #include <stdio.h> |
-+#include <time.h> |
-+#ifndef CLK_TCK |
-+#define CLK_TCK CLOCKS_PER_SEC |
-+#endif |
-+ |
- #endif |
- |
- U_NAMESPACE_BEGIN |
-@@ -45,6 +51,11 @@ |
- * MutableTrieDictionary |
- */ |
- |
-+//#define MAX_VALUE 65535 |
-+ |
-+// forward declaration |
-+inline uint16_t scaleLogProbabilities(double logprob); |
-+ |
- // Node structure for the ternary, uncompressed trie |
- struct TernaryNode : public UMemory { |
- UChar ch; // UTF-16 code unit |
-@@ -77,7 +88,8 @@ |
- delete high; |
- } |
- |
--MutableTrieDictionary::MutableTrieDictionary( UChar median, UErrorCode &status ) { |
-+MutableTrieDictionary::MutableTrieDictionary( UChar median, UErrorCode &status, |
-+ UBool containsValue /* = FALSE */ ) { |
- // Start the trie off with something. Having the root node already present |
- // cuts a special case out of the search/insertion functions. |
- // Making it a median character cuts the worse case for searches from |
-@@ -91,14 +103,19 @@ |
- if (U_SUCCESS(status) && fIter == NULL) { |
- status = U_MEMORY_ALLOCATION_ERROR; |
- } |
-+ |
-+ fValued = containsValue; |
- } |
- |
--MutableTrieDictionary::MutableTrieDictionary( UErrorCode &status ) { |
-+MutableTrieDictionary::MutableTrieDictionary( UErrorCode &status, |
-+ UBool containsValue /* = false */ ) { |
- fTrie = NULL; |
- fIter = utext_openUChars(NULL, NULL, 0, &status); |
- if (U_SUCCESS(status) && fIter == NULL) { |
- status = U_MEMORY_ALLOCATION_ERROR; |
- } |
-+ |
-+ fValued = containsValue; |
- } |
- |
- MutableTrieDictionary::~MutableTrieDictionary() { |
-@@ -108,12 +125,13 @@ |
- |
- int32_t |
- MutableTrieDictionary::search( UText *text, |
-- int32_t maxLength, |
-- int32_t *lengths, |
-- int &count, |
-- int limit, |
-- TernaryNode *&parent, |
-- UBool &pMatched ) const { |
-+ int32_t maxLength, |
-+ int32_t *lengths, |
-+ int &count, |
-+ int limit, |
-+ TernaryNode *&parent, |
-+ UBool &pMatched, |
-+ uint16_t *values /*=NULL*/) const { |
- // TODO: current implementation works in UTF-16 space |
- const TernaryNode *up = NULL; |
- const TernaryNode *p = fTrie; |
-@@ -121,6 +139,10 @@ |
- pMatched = TRUE; |
- int i; |
- |
-+ if (!fValued) { |
-+ values = NULL; |
-+ } |
-+ |
- UChar uc = utext_current32(text); |
- for (i = 0; i < maxLength && p != NULL; ++i) { |
- while (p != NULL) { |
-@@ -141,7 +163,11 @@ |
- break; |
- } |
- // Must be equal to get here |
-- if (limit > 0 && (p->flags & kEndsWord)) { |
-+ if (limit > 0 && (p->flags > 0)) { |
-+ //is there a more efficient way to add values? ie. remove if stmt |
-+ if(values != NULL) { |
-+ values[mycount] = p->flags; |
-+ } |
- lengths[mycount++] = i+1; |
- --limit; |
- } |
-@@ -161,13 +187,14 @@ |
- void |
- MutableTrieDictionary::addWord( const UChar *word, |
- int32_t length, |
-- UErrorCode &status ) { |
--#if 0 |
-- if (length <= 0) { |
-+ UErrorCode &status, |
-+ uint16_t value /* = 0 */ ) { |
-+ // dictionary cannot store zero values, would interfere with flags |
-+ if (length <= 0 || (!fValued && value > 0) || (fValued && value == 0)) { |
- status = U_ILLEGAL_ARGUMENT_ERROR; |
- return; |
- } |
--#endif |
-+ |
- TernaryNode *parent; |
- UBool pMatched; |
- int count; |
-@@ -177,7 +204,7 @@ |
- matched = search(fIter, length, NULL, count, 0, parent, pMatched); |
- |
- while (matched++ < length) { |
-- UChar32 uc = utext_next32(fIter); // TODO: supplemetary support? |
-+ UChar32 uc = utext_next32(fIter); // TODO: supplementary support? |
- U_ASSERT(uc != U_SENTINEL); |
- TernaryNode *newNode = new TernaryNode(uc); |
- if (newNode == NULL) { |
-@@ -199,30 +226,23 @@ |
- parent = newNode; |
- } |
- |
-- parent->flags |= kEndsWord; |
--} |
-- |
--#if 0 |
--void |
--MutableTrieDictionary::addWords( UEnumeration *words, |
-- UErrorCode &status ) { |
-- int32_t length; |
-- const UChar *word; |
-- while ((word = uenum_unext(words, &length, &status)) && U_SUCCESS(status)) { |
-- addWord(word, length, status); |
-+ if(fValued && value > 0){ |
-+ parent->flags = value; |
-+ } else { |
-+ parent->flags |= kEndsWord; |
- } |
- } |
--#endif |
- |
- int32_t |
- MutableTrieDictionary::matches( UText *text, |
- int32_t maxLength, |
- int32_t *lengths, |
- int &count, |
-- int limit ) const { |
-+ int limit, |
-+ uint16_t *values /*=NULL*/) const { |
- TernaryNode *parent; |
- UBool pMatched; |
-- return search(text, maxLength, lengths, count, limit, parent, pMatched); |
-+ return search(text, maxLength, lengths, count, limit, parent, pMatched, values); |
- } |
- |
- // Implementation of iteration for MutableTrieDictionary |
-@@ -277,7 +297,7 @@ |
- break; |
- } |
- case kEqual: |
-- emit = (node->flags & kEndsWord) != 0; |
-+ emit = node->flags > 0; |
- equal = (node->equal != NULL); |
- // If this node should be part of the next emitted string, append |
- // the UChar to the string, and make sure we pop it when we come |
-@@ -299,7 +319,7 @@ |
- } |
- case kGreaterThan: |
- // If this node's character is in the string, remove it. |
-- if (node->equal != NULL || (node->flags & kEndsWord)) { |
-+ if (node->equal != NULL || node->flags > 0) { |
- unistr.truncate(unistr.length()-1); |
- } |
- if (node->high != NULL) { |
-@@ -354,12 +374,75 @@ |
- * CompactTrieDictionary |
- */ |
- |
-+//TODO further optimization: |
-+// minimise size of trie with logprobs by storing values |
-+// for terminal nodes directly in offsets[] |
-+// --> calculating from next offset *might* be simpler, but would have to add |
-+// one last offset for logprob of last node |
-+// --> if calculate from current offset, need to factor in possible overflow |
-+// as well. |
-+// idea: store in offset, set first bit to indicate logprob storage-->won't |
-+// have to access additional node |
-+ |
-+// {'Dic', 1}, version 1: uses old header, no values |
-+#define COMPACT_TRIE_MAGIC_1 0x44696301 |
-+// version 2: uses new header (more than 2^16 nodes), no values |
-+#define COMPACT_TRIE_MAGIC_2 0x44696302 |
-+// version 3: uses new header, includes values |
-+#define COMPACT_TRIE_MAGIC_3 0x44696303 |
-+ |
- struct CompactTrieHeader { |
- uint32_t size; // Size of the data in bytes |
- uint32_t magic; // Magic number (including version) |
-+ uint32_t nodeCount; // Number of entries in offsets[] |
-+ uint32_t root; // Node number of the root node |
-+ uint32_t offsets[1]; // Offsets to nodes from start of data |
-+}; |
-+ |
-+// old version of CompactTrieHeader kept for backwards compatibility |
-+struct CompactTrieHeaderV1 { |
-+ uint32_t size; // Size of the data in bytes |
-+ uint32_t magic; // Magic number (including version) |
- uint16_t nodeCount; // Number of entries in offsets[] |
- uint16_t root; // Node number of the root node |
-- uint32_t offsets[1]; // Offsets to nodes from start of data |
-+ uint32_t offsets[1]; // Offsets to nodes from start of data |
-+}; |
-+ |
-+// Helper class for managing CompactTrieHeader and CompactTrieHeaderV1 |
-+struct CompactTrieInfo { |
-+ uint32_t size; // Size of the data in bytes |
-+ uint32_t magic; // Magic number (including version) |
-+ uint32_t nodeCount; // Number of entries in offsets[] |
-+ uint32_t root; // Node number of the root node |
-+ uint32_t *offsets; // Offsets to nodes from start of data |
-+ uint8_t *address; // pointer to header bytes in memory |
-+ |
-+ CompactTrieInfo(const void *data, UErrorCode &status){ |
-+ CompactTrieHeader *header = (CompactTrieHeader *) data; |
-+ if (header->magic != COMPACT_TRIE_MAGIC_1 && |
-+ header->magic != COMPACT_TRIE_MAGIC_2 && |
-+ header->magic != COMPACT_TRIE_MAGIC_3) { |
-+ status = U_ILLEGAL_ARGUMENT_ERROR; |
-+ } else { |
-+ size = header->size; |
-+ magic = header->magic; |
-+ |
-+ if (header->magic == COMPACT_TRIE_MAGIC_1) { |
-+ CompactTrieHeaderV1 *headerV1 = (CompactTrieHeaderV1 *) header; |
-+ nodeCount = headerV1->nodeCount; |
-+ root = headerV1->root; |
-+ offsets = &(headerV1->offsets[0]); |
-+ address = (uint8_t *)headerV1; |
-+ } else { |
-+ nodeCount = header->nodeCount; |
-+ root = header->root; |
-+ offsets = &(header->offsets[0]); |
-+ address = (uint8_t *)header; |
-+ } |
-+ } |
-+ } |
-+ |
-+ ~CompactTrieInfo(){} |
- }; |
- |
- // Note that to avoid platform-specific alignment issues, all members of the node |
-@@ -375,10 +458,14 @@ |
- enum CompactTrieNodeFlags { |
- kVerticalNode = 0x1000, // This is a vertical node |
- kParentEndsWord = 0x2000, // The node whose equal link points to this ends a word |
-- kReservedFlag1 = 0x4000, |
-- kReservedFlag2 = 0x8000, |
-+ kExceedsCount = 0x4000, // new MSB for count >= 4096, originally kReservedFlag1 |
-+ kEqualOverflows = 0x8000, // Links to nodeIDs > 2^16, orig. kReservedFlag2 |
- kCountMask = 0x0FFF, // The count portion of flagscount |
-- kFlagMask = 0xF000 // The flags portion of flagscount |
-+ kFlagMask = 0xF000, // The flags portion of flagscount |
-+ kRootCountMask = 0x7FFF // The count portion of flagscount in the root node |
-+ |
-+ //offset flags: |
-+ //kOffsetContainsValue = 0x80000000 // Offset contains value for parent node |
- }; |
- |
- // The two node types are distinguished by the kVerticalNode flag. |
-@@ -402,63 +489,177 @@ |
- uint16_t chars[1]; // Code units |
- }; |
- |
--// {'Dic', 1}, version 1 |
--#define COMPACT_TRIE_MAGIC_1 0x44696301 |
-- |
- CompactTrieDictionary::CompactTrieDictionary(UDataMemory *dataObj, |
- UErrorCode &status ) |
- : fUData(dataObj) |
- { |
-- fData = (const CompactTrieHeader *) udata_getMemory(dataObj); |
-+ fInfo = (CompactTrieInfo *)uprv_malloc(sizeof(CompactTrieInfo)); |
-+ *fInfo = CompactTrieInfo(udata_getMemory(dataObj), status); |
- fOwnData = FALSE; |
-- if (fData->magic != COMPACT_TRIE_MAGIC_1) { |
-- status = U_ILLEGAL_ARGUMENT_ERROR; |
-- fData = NULL; |
-- } |
- } |
-+ |
- CompactTrieDictionary::CompactTrieDictionary( const void *data, |
- UErrorCode &status ) |
- : fUData(NULL) |
- { |
-- fData = (const CompactTrieHeader *) data; |
-+ fInfo = (CompactTrieInfo *)uprv_malloc(sizeof(CompactTrieInfo)); |
-+ *fInfo = CompactTrieInfo(data, status); |
- fOwnData = FALSE; |
-- if (fData->magic != COMPACT_TRIE_MAGIC_1) { |
-- status = U_ILLEGAL_ARGUMENT_ERROR; |
-- fData = NULL; |
-- } |
- } |
- |
- CompactTrieDictionary::CompactTrieDictionary( const MutableTrieDictionary &dict, |
- UErrorCode &status ) |
- : fUData(NULL) |
- { |
-- fData = compactMutableTrieDictionary(dict, status); |
-+ const CompactTrieHeader* header = compactMutableTrieDictionary(dict, status); |
-+ if (U_SUCCESS(status)) { |
-+ fInfo = (CompactTrieInfo *)uprv_malloc(sizeof(CompactTrieInfo)); |
-+ *fInfo = CompactTrieInfo(header, status); |
-+ } |
-+ |
- fOwnData = !U_FAILURE(status); |
- } |
- |
- CompactTrieDictionary::~CompactTrieDictionary() { |
- if (fOwnData) { |
-- uprv_free((void *)fData); |
-+ uprv_free((void *)(fInfo->address)); |
- } |
-+ uprv_free((void *)fInfo); |
-+ |
- if (fUData) { |
- udata_close(fUData); |
- } |
- } |
- |
-+UBool CompactTrieDictionary::getValued() const{ |
-+ return fInfo->magic == COMPACT_TRIE_MAGIC_3; |
-+} |
-+ |
- uint32_t |
- CompactTrieDictionary::dataSize() const { |
-- return fData->size; |
-+ return fInfo->size; |
- } |
- |
- const void * |
- CompactTrieDictionary::data() const { |
-- return fData; |
-+ return fInfo->address; |
-+} |
-+ |
-+//This function finds the address of a node for us, given its node ID |
-+static inline const CompactTrieNode * |
-+getCompactNode(const CompactTrieInfo *info, uint32_t node) { |
-+ if(node < info->root-1) { |
-+ return (const CompactTrieNode *)(&info->offsets[node]); |
-+ } else { |
-+ return (const CompactTrieNode *)(info->address + info->offsets[node]); |
-+ } |
- } |
- |
--// This function finds the address of a node for us, given its node ID |
-+//this version of getCompactNode is currently only used in compactMutableTrieDictionary() |
- static inline const CompactTrieNode * |
--getCompactNode(const CompactTrieHeader *header, uint16_t node) { |
-- return (const CompactTrieNode *)((const uint8_t *)header + header->offsets[node]); |
-+getCompactNode(const CompactTrieHeader *header, uint32_t node) { |
-+ if(node < header->root-1) { |
-+ return (const CompactTrieNode *)(&header->offsets[node]); |
-+ } else { |
-+ return (const CompactTrieNode *)((const uint8_t *)header + header->offsets[node]); |
-+ } |
-+} |
-+ |
-+ |
-+/** |
-+ * Calculates the number of links in a node |
-+ * @node The specified node |
-+ */ |
-+static inline const uint16_t |
-+getCount(const CompactTrieNode *node){ |
-+ return (node->flagscount & kCountMask); |
-+ //use the code below if number of links ever exceed 4096 |
-+ //return (node->flagscount & kCountMask) + ((node->flagscount & kExceedsCount) >> 2); |
-+} |
-+ |
-+/** |
-+ * calculates an equal link node ID of a horizontal node |
-+ * @hnode The horizontal node containing the equal link |
-+ * @param index The index into hnode->entries[] |
-+ * @param nodeCount The length of hnode->entries[] |
-+ */ |
-+static inline uint32_t calcEqualLink(const CompactTrieVerticalNode *vnode){ |
-+ if(vnode->flagscount & kEqualOverflows){ |
-+ // treat overflow bits as an extension of chars[] |
-+ uint16_t *overflow = (uint16_t *) &vnode->chars[getCount((CompactTrieNode*)vnode)]; |
-+ return vnode->equal + (((uint32_t)*overflow) << 16); |
-+ }else{ |
-+ return vnode->equal; |
-+ } |
-+} |
-+ |
-+/** |
-+ * calculates an equal link node ID of a horizontal node |
-+ * @hnode The horizontal node containing the equal link |
-+ * @param index The index into hnode->entries[] |
-+ * @param nodeCount The length of hnode->entries[] |
-+ */ |
-+static inline uint32_t calcEqualLink(const CompactTrieHorizontalNode *hnode, uint16_t index, uint16_t nodeCount){ |
-+ if(hnode->flagscount & kEqualOverflows){ |
-+ //set overflow to point to the uint16_t containing the overflow bits |
-+ uint16_t *overflow = (uint16_t *) &hnode->entries[nodeCount]; |
-+ overflow += index/4; |
-+ uint16_t extraBits = (*overflow >> (3 - (index % 4)) * 4) % 0x10; |
-+ return hnode->entries[index].equal + (((uint32_t)extraBits) << 16); |
-+ } else { |
-+ return hnode->entries[index].equal; |
-+ } |
-+} |
-+ |
-+/** |
-+ * Returns the value stored in the specified node which is associated with its |
-+ * parent node. |
-+ * TODO: how to tell that value is stored in node or in offset? check whether |
-+ * node ID < fInfo->root! |
-+ */ |
-+static inline uint16_t getValue(const CompactTrieHorizontalNode *hnode){ |
-+ uint16_t count = getCount((CompactTrieNode *)hnode); |
-+ uint16_t overflowSize = 0; //size of node ID overflow storage in bytes |
-+ |
-+ if(hnode->flagscount & kEqualOverflows) |
-+ overflowSize = (count + 3) / 4 * sizeof(uint16_t); |
-+ return *((uint16_t *)((uint8_t *)&hnode->entries[count] + overflowSize)); |
-+} |
-+ |
-+static inline uint16_t getValue(const CompactTrieVerticalNode *vnode){ |
-+ // calculate size of total node ID overflow storage in bytes |
-+ uint16_t overflowSize = (vnode->flagscount & kEqualOverflows)? sizeof(uint16_t) : 0; |
-+ return *((uint16_t *)((uint8_t *)&vnode->chars[getCount((CompactTrieNode *)vnode)] + overflowSize)); |
-+} |
-+ |
-+static inline uint16_t getValue(const CompactTrieNode *node){ |
-+ if(node->flagscount & kVerticalNode) |
-+ return getValue((const CompactTrieVerticalNode *)node); |
-+ else |
-+ return getValue((const CompactTrieHorizontalNode *)node); |
-+} |
-+ |
-+//returns index of match in CompactTrieHorizontalNode.entries[] using binary search |
-+inline int16_t |
-+searchHorizontalEntries(const CompactTrieHorizontalEntry *entries, |
-+ UChar uc, uint16_t nodeCount){ |
-+ int low = 0; |
-+ int high = nodeCount-1; |
-+ int middle; |
-+ while (high >= low) { |
-+ middle = (high+low)/2; |
-+ if (uc == entries[middle].ch) { |
-+ return middle; |
-+ } |
-+ else if (uc < entries[middle].ch) { |
-+ high = middle-1; |
-+ } |
-+ else { |
-+ low = middle+1; |
-+ } |
-+ } |
-+ |
-+ return -1; |
- } |
- |
- int32_t |
-@@ -466,17 +667,38 @@ |
- int32_t maxLength, |
- int32_t *lengths, |
- int &count, |
-- int limit ) const { |
-+ int limit, |
-+ uint16_t *values /*= NULL*/) const { |
-+ if (fInfo->magic == COMPACT_TRIE_MAGIC_2) |
-+ values = NULL; |
-+ |
- // TODO: current implementation works in UTF-16 space |
-- const CompactTrieNode *node = getCompactNode(fData, fData->root); |
-+ const CompactTrieNode *node = getCompactNode(fInfo, fInfo->root); |
- int mycount = 0; |
- |
- UChar uc = utext_current32(text); |
- int i = 0; |
- |
-+ // handle root node with only kEqualOverflows flag: assume horizontal node without parent |
-+ if(node != NULL){ |
-+ const CompactTrieHorizontalNode *root = (const CompactTrieHorizontalNode *) node; |
-+ int index = searchHorizontalEntries(root->entries, uc, root->flagscount & kRootCountMask); |
-+ if(index > -1){ |
-+ node = getCompactNode(fInfo, calcEqualLink(root, index, root->flagscount & kRootCountMask)); |
-+ utext_next32(text); |
-+ uc = utext_current32(text); |
-+ ++i; |
-+ }else{ |
-+ node = NULL; |
-+ } |
-+ } |
-+ |
- while (node != NULL) { |
- // Check if the node we just exited ends a word |
- if (limit > 0 && (node->flagscount & kParentEndsWord)) { |
-+ if(values != NULL){ |
-+ values[mycount] = getValue(node); |
-+ } |
- lengths[mycount++] = i; |
- --limit; |
- } |
-@@ -487,7 +709,7 @@ |
- break; |
- } |
- |
-- int nodeCount = (node->flagscount & kCountMask); |
-+ int nodeCount = getCount(node); |
- if (nodeCount == 0) { |
- // Special terminal node; return now |
- break; |
-@@ -507,35 +729,27 @@ |
- // To get here we must have come through the whole list successfully; |
- // go on to the next node. Note that a word cannot end in the middle |
- // of a vertical node. |
-- node = getCompactNode(fData, vnode->equal); |
-+ node = getCompactNode(fInfo, calcEqualLink(vnode)); |
- } |
- else { |
- // Horizontal node; do binary search |
- const CompactTrieHorizontalNode *hnode = (const CompactTrieHorizontalNode *)node; |
-- int low = 0; |
-- int high = nodeCount-1; |
-- int middle; |
-- node = NULL; // If we don't find a match, we'll fall out of the loop |
-- while (high >= low) { |
-- middle = (high+low)/2; |
-- if (uc == hnode->entries[middle].ch) { |
-- // We hit a match; get the next node and next character |
-- node = getCompactNode(fData, hnode->entries[middle].equal); |
-- utext_next32(text); |
-- uc = utext_current32(text); |
-- ++i; |
-- break; |
-- } |
-- else if (uc < hnode->entries[middle].ch) { |
-- high = middle-1; |
-- } |
-- else { |
-- low = middle+1; |
-- } |
-+ const CompactTrieHorizontalEntry *entries; |
-+ entries = hnode->entries; |
-+ |
-+ int index = searchHorizontalEntries(entries, uc, nodeCount); |
-+ if(index > -1){ // |
-+ // We hit a match; get the next node and next character |
-+ node = getCompactNode(fInfo, calcEqualLink(hnode, index, nodeCount)); |
-+ utext_next32(text); |
-+ uc = utext_current32(text); |
-+ ++i; |
-+ }else{ |
-+ node = NULL; // If we don't find a match, we'll fall out of the loop |
- } |
- } |
- } |
--exit: |
-+ exit: |
- count = mycount; |
- return i; |
- } |
-@@ -545,16 +759,16 @@ |
- private: |
- UVector32 fNodeStack; // Stack of nodes to process |
- UVector32 fIndexStack; // Stack of where in node we are |
-- const CompactTrieHeader *fHeader; // Trie data |
-+ const CompactTrieInfo *fInfo; // Trie data |
- |
- public: |
- static UClassID U_EXPORT2 getStaticClassID(void); |
- virtual UClassID getDynamicClassID(void) const; |
- public: |
-- CompactTrieEnumeration(const CompactTrieHeader *header, UErrorCode &status) |
-+ CompactTrieEnumeration(const CompactTrieInfo *info, UErrorCode &status) |
- : fNodeStack(status), fIndexStack(status) { |
-- fHeader = header; |
-- fNodeStack.push(header->root, status); |
-+ fInfo = info; |
-+ fNodeStack.push(info->root, status); |
- fIndexStack.push(0, status); |
- unistr.remove(); |
- } |
-@@ -564,14 +778,14 @@ |
- |
- virtual StringEnumeration *clone() const { |
- UErrorCode status = U_ZERO_ERROR; |
-- return new CompactTrieEnumeration(fHeader, status); |
-+ return new CompactTrieEnumeration(fInfo, status); |
- } |
- |
- virtual const UnicodeString * snext(UErrorCode &status); |
- |
- // Very expensive, but this should never be used. |
- virtual int32_t count(UErrorCode &status) const { |
-- CompactTrieEnumeration counter(fHeader, status); |
-+ CompactTrieEnumeration counter(fInfo, status); |
- int32_t result = 0; |
- while (counter.snext(status) != NULL && U_SUCCESS(status)) { |
- ++result; |
-@@ -582,7 +796,7 @@ |
- virtual void reset(UErrorCode &status) { |
- fNodeStack.removeAllElements(); |
- fIndexStack.removeAllElements(); |
-- fNodeStack.push(fHeader->root, status); |
-+ fNodeStack.push(fInfo->root, status); |
- fIndexStack.push(0, status); |
- unistr.remove(); |
- } |
-@@ -595,26 +809,34 @@ |
- if (fNodeStack.empty() || U_FAILURE(status)) { |
- return NULL; |
- } |
-- const CompactTrieNode *node = getCompactNode(fHeader, fNodeStack.peeki()); |
-+ const CompactTrieNode *node = getCompactNode(fInfo, fNodeStack.peeki()); |
- int where = fIndexStack.peeki(); |
- while (!fNodeStack.empty() && U_SUCCESS(status)) { |
-- int nodeCount = (node->flagscount & kCountMask); |
-+ int nodeCount; |
-+ |
-+ bool isRoot = fNodeStack.peeki() == static_cast<int32_t>(fInfo->root); |
-+ if(isRoot){ |
-+ nodeCount = node->flagscount & kRootCountMask; |
-+ } else { |
-+ nodeCount = getCount(node); |
-+ } |
-+ |
- UBool goingDown = FALSE; |
- if (nodeCount == 0) { |
- // Terminal node; go up immediately |
- fNodeStack.popi(); |
- fIndexStack.popi(); |
-- node = getCompactNode(fHeader, fNodeStack.peeki()); |
-+ node = getCompactNode(fInfo, fNodeStack.peeki()); |
- where = fIndexStack.peeki(); |
- } |
-- else if (node->flagscount & kVerticalNode) { |
-+ else if ((node->flagscount & kVerticalNode) && !isRoot) { |
- // Vertical node |
- const CompactTrieVerticalNode *vnode = (const CompactTrieVerticalNode *)node; |
- if (where == 0) { |
- // Going down |
-- unistr.append((const UChar *)vnode->chars, (int32_t) nodeCount); |
-+ unistr.append((const UChar *)vnode->chars, nodeCount); |
- fIndexStack.setElementAt(1, fIndexStack.size()-1); |
-- node = getCompactNode(fHeader, fNodeStack.push(vnode->equal, status)); |
-+ node = getCompactNode(fInfo, fNodeStack.push(calcEqualLink(vnode), status)); |
- where = fIndexStack.push(0, status); |
- goingDown = TRUE; |
- } |
-@@ -623,7 +845,7 @@ |
- unistr.truncate(unistr.length()-nodeCount); |
- fNodeStack.popi(); |
- fIndexStack.popi(); |
-- node = getCompactNode(fHeader, fNodeStack.peeki()); |
-+ node = getCompactNode(fInfo, fNodeStack.peeki()); |
- where = fIndexStack.peeki(); |
- } |
- } |
-@@ -638,7 +860,7 @@ |
- // Push on next node |
- unistr.append((UChar)hnode->entries[where].ch); |
- fIndexStack.setElementAt(where+1, fIndexStack.size()-1); |
-- node = getCompactNode(fHeader, fNodeStack.push(hnode->entries[where].equal, status)); |
-+ node = getCompactNode(fInfo, fNodeStack.push(calcEqualLink(hnode, where, nodeCount), status)); |
- where = fIndexStack.push(0, status); |
- goingDown = TRUE; |
- } |
-@@ -646,12 +868,14 @@ |
- // Going up |
- fNodeStack.popi(); |
- fIndexStack.popi(); |
-- node = getCompactNode(fHeader, fNodeStack.peeki()); |
-+ node = getCompactNode(fInfo, fNodeStack.peeki()); |
- where = fIndexStack.peeki(); |
- } |
- } |
-+ |
- // Check if the parent of the node we've just gone down to ends a |
- // word. If so, return it. |
-+ // The root node should never end up here. |
- if (goingDown && (node->flagscount & kParentEndsWord)) { |
- return &unistr; |
- } |
-@@ -664,7 +888,7 @@ |
- if (U_FAILURE(status)) { |
- return NULL; |
- } |
-- return new CompactTrieEnumeration(fData, status); |
-+ return new CompactTrieEnumeration(fInfo, status); |
- } |
- |
- // |
-@@ -672,21 +896,36 @@ |
- // and back again |
- // |
- |
--// Helper classes to construct the compact trie |
-+enum CompactTrieNodeType { |
-+ kHorizontalType = 0, |
-+ kVerticalType = 1, |
-+ kValueType = 2 |
-+}; |
-+ |
-+/** |
-+ * The following classes (i.e. BuildCompactTrie*Node) are helper classes to |
-+ * construct the compact trie by storing information for each node and later |
-+ * writing the node to memory in a sequential format. |
-+ */ |
- class BuildCompactTrieNode: public UMemory { |
-- public: |
-+public: |
- UBool fParentEndsWord; |
-- UBool fVertical; |
-+ CompactTrieNodeType fNodeType; |
- UBool fHasDuplicate; |
-+ UBool fEqualOverflows; |
- int32_t fNodeID; |
- UnicodeString fChars; |
-+ uint16_t fValue; |
- |
-- public: |
-- BuildCompactTrieNode(UBool parentEndsWord, UBool vertical, UStack &nodes, UErrorCode &status) { |
-+public: |
-+ BuildCompactTrieNode(UBool parentEndsWord, CompactTrieNodeType nodeType, |
-+ UStack &nodes, UErrorCode &status, uint16_t value = 0) { |
- fParentEndsWord = parentEndsWord; |
- fHasDuplicate = FALSE; |
-- fVertical = vertical; |
-+ fNodeType = nodeType; |
-+ fEqualOverflows = FALSE; |
- fNodeID = nodes.size(); |
-+ fValue = parentEndsWord? value : 0; |
- nodes.push(this, status); |
- } |
- |
-@@ -694,87 +933,225 @@ |
- } |
- |
- virtual uint32_t size() { |
-- return sizeof(uint16_t); |
-+ if(fValue > 0) |
-+ return sizeof(uint16_t) * 2; |
-+ else |
-+ return sizeof(uint16_t); |
- } |
- |
- virtual void write(uint8_t *bytes, uint32_t &offset, const UVector32 &/*translate*/) { |
- // Write flag/count |
-- *((uint16_t *)(bytes+offset)) = (fChars.length() & kCountMask) |
-- | (fVertical ? kVerticalNode : 0) | (fParentEndsWord ? kParentEndsWord : 0 ); |
-+ |
-+ // if this ever fails, a flag bit (i.e. kExceedsCount) will need to be |
-+ // used as a 5th MSB. |
-+ U_ASSERT(fChars.length() < 4096 || fNodeID == 2); |
-+ |
-+ *((uint16_t *)(bytes+offset)) = (fEqualOverflows? kEqualOverflows : 0) | |
-+ ((fNodeID == 2)? (fChars.length() & kRootCountMask): |
-+ ( |
-+ (fChars.length() & kCountMask) | |
-+ //((fChars.length() << 2) & kExceedsCount) | |
-+ (fNodeType == kVerticalType ? kVerticalNode : 0) | |
-+ (fParentEndsWord ? kParentEndsWord : 0 ) |
-+ ) |
-+ ); |
- offset += sizeof(uint16_t); |
- } |
-+ |
-+ virtual void writeValue(uint8_t *bytes, uint32_t &offset) { |
-+ if(fValue > 0){ |
-+ *((uint16_t *)(bytes+offset)) = fValue; |
-+ offset += sizeof(uint16_t); |
-+ } |
-+ } |
-+ |
-+}; |
-+ |
-+/** |
-+ * Stores value of parent terminating nodes that have no more subtries. |
-+ */ |
-+class BuildCompactTrieValueNode: public BuildCompactTrieNode { |
-+public: |
-+ BuildCompactTrieValueNode(UStack &nodes, UErrorCode &status, uint16_t value) |
-+ : BuildCompactTrieNode(TRUE, kValueType, nodes, status, value){ |
-+ } |
-+ |
-+ virtual ~BuildCompactTrieValueNode(){ |
-+ } |
-+ |
-+ virtual uint32_t size() { |
-+ return sizeof(uint16_t) * 2; |
-+ } |
-+ |
-+ virtual void write(uint8_t *bytes, uint32_t &offset, const UVector32 &translate) { |
-+ // don't write value directly to memory but store it in offset to be written later |
-+ //offset = fValue & kOffsetContainsValue; |
-+ BuildCompactTrieNode::write(bytes, offset, translate); |
-+ BuildCompactTrieNode::writeValue(bytes, offset); |
-+ } |
- }; |
- |
- class BuildCompactTrieHorizontalNode: public BuildCompactTrieNode { |
- public: |
- UStack fLinks; |
-+ UBool fMayOverflow; //intermediate value for fEqualOverflows |
- |
- public: |
-- BuildCompactTrieHorizontalNode(UBool parentEndsWord, UStack &nodes, UErrorCode &status) |
-- : BuildCompactTrieNode(parentEndsWord, FALSE, nodes, status), fLinks(status) { |
-+ BuildCompactTrieHorizontalNode(UBool parentEndsWord, UStack &nodes, UErrorCode &status, uint16_t value = 0) |
-+ : BuildCompactTrieNode(parentEndsWord, kHorizontalType, nodes, status, value), fLinks(status) { |
-+ fMayOverflow = FALSE; |
- } |
- |
- virtual ~BuildCompactTrieHorizontalNode() { |
- } |
- |
-+ // It is impossible to know beforehand exactly how much space the node will |
-+ // need in memory before being written, because the node IDs in the equal |
-+ // links may or may not overflow after node coalescing. Therefore, this method |
-+ // returns the maximum size possible for the node. |
- virtual uint32_t size() { |
-- return offsetof(CompactTrieHorizontalNode,entries) + |
-- (fChars.length()*sizeof(CompactTrieHorizontalEntry)); |
-+ uint32_t estimatedSize = offsetof(CompactTrieHorizontalNode,entries) + |
-+ (fChars.length()*sizeof(CompactTrieHorizontalEntry)); |
-+ |
-+ if(fValue > 0) |
-+ estimatedSize += sizeof(uint16_t); |
-+ |
-+ //estimate extra space needed to store overflow for node ID links |
-+ //may be more than what is actually needed |
-+ for(int i=0; i < fChars.length(); i++){ |
-+ if(((BuildCompactTrieNode *)fLinks[i])->fNodeID > 0xFFFF){ |
-+ fMayOverflow = TRUE; |
-+ break; |
-+ } |
-+ } |
-+ if(fMayOverflow) // added space for overflow should be same as ceil(fChars.length()/4) * sizeof(uint16_t) |
-+ estimatedSize += (sizeof(uint16_t) * fChars.length() + 2)/4; |
-+ |
-+ return estimatedSize; |
- } |
- |
- virtual void write(uint8_t *bytes, uint32_t &offset, const UVector32 &translate) { |
-- BuildCompactTrieNode::write(bytes, offset, translate); |
- int32_t count = fChars.length(); |
-+ |
-+ //if largest nodeID > 2^16, set flag |
-+ //large node IDs are more likely to be at the back of the array |
-+ for (int32_t i = count-1; i >= 0; --i) { |
-+ if(translate.elementAti(((BuildCompactTrieNode *)fLinks[i])->fNodeID) > 0xFFFF){ |
-+ fEqualOverflows = TRUE; |
-+ break; |
-+ } |
-+ } |
-+ |
-+ BuildCompactTrieNode::write(bytes, offset, translate); |
-+ |
-+ // write entries[] to memory |
- for (int32_t i = 0; i < count; ++i) { |
- CompactTrieHorizontalEntry *entry = (CompactTrieHorizontalEntry *)(bytes+offset); |
- entry->ch = fChars[i]; |
- entry->equal = translate.elementAti(((BuildCompactTrieNode *)fLinks[i])->fNodeID); |
- #ifdef DEBUG_TRIE_DICT |
-- if (entry->equal == 0) { |
-+ |
-+ if ((entry->equal == 0) && !fEqualOverflows) { |
- fprintf(stderr, "ERROR: horizontal link %d, logical node %d maps to physical node zero\n", |
- i, ((BuildCompactTrieNode *)fLinks[i])->fNodeID); |
- } |
- #endif |
- offset += sizeof(CompactTrieHorizontalEntry); |
- } |
-+ |
-+ // append extra bits of equal nodes to end if fEqualOverflows |
-+ if (fEqualOverflows) { |
-+ uint16_t leftmostBits = 0; |
-+ for (int16_t i = 0; i < count; i++) { |
-+ leftmostBits = (leftmostBits << 4) | getLeftmostBits(translate, i); |
-+ |
-+ // write filled uint16_t to memory |
-+ if(i % 4 == 3){ |
-+ *((uint16_t *)(bytes+offset)) = leftmostBits; |
-+ leftmostBits = 0; |
-+ offset += sizeof(uint16_t); |
-+ } |
-+ } |
-+ |
-+ // pad last uint16_t with zeroes if necessary |
-+ int remainder = count % 4; |
-+ if (remainder > 0) { |
-+ *((uint16_t *)(bytes+offset)) = (leftmostBits << (16 - 4 * remainder)); |
-+ offset += sizeof(uint16_t); |
-+ } |
-+ } |
-+ |
-+ BuildCompactTrieNode::writeValue(bytes, offset); |
-+ } |
-+ |
-+ // returns leftmost bits of physical node link |
-+ uint16_t getLeftmostBits(const UVector32 &translate, uint32_t i){ |
-+ uint16_t leftmostBits = (uint16_t) (translate.elementAti(((BuildCompactTrieNode *)fLinks[i])->fNodeID) >> 16); |
-+#ifdef DEBUG_TRIE_DICT |
-+ if (leftmostBits > 0xF) { |
-+ fprintf(stderr, "ERROR: horizontal link %d, logical node %d exceeds maximum possible node ID value\n", |
-+ i, ((BuildCompactTrieNode *)fLinks[i])->fNodeID); |
-+ } |
-+#endif |
-+ return leftmostBits; |
- } |
- |
- void addNode(UChar ch, BuildCompactTrieNode *link, UErrorCode &status) { |
- fChars.append(ch); |
- fLinks.push(link, status); |
- } |
-+ |
- }; |
- |
- class BuildCompactTrieVerticalNode: public BuildCompactTrieNode { |
-- public: |
-+public: |
- BuildCompactTrieNode *fEqual; |
- |
-- public: |
-- BuildCompactTrieVerticalNode(UBool parentEndsWord, UStack &nodes, UErrorCode &status) |
-- : BuildCompactTrieNode(parentEndsWord, TRUE, nodes, status) { |
-+public: |
-+ BuildCompactTrieVerticalNode(UBool parentEndsWord, UStack &nodes, UErrorCode &status, uint16_t value = 0) |
-+ : BuildCompactTrieNode(parentEndsWord, kVerticalType, nodes, status, value) { |
- fEqual = NULL; |
- } |
- |
- virtual ~BuildCompactTrieVerticalNode() { |
- } |
- |
-+ // Returns the maximum possible size of this node. See comment in |
-+ // BuildCompactTrieHorizontal node for more information. |
- virtual uint32_t size() { |
-- return offsetof(CompactTrieVerticalNode,chars) + (fChars.length()*sizeof(uint16_t)); |
-+ uint32_t estimatedSize = offsetof(CompactTrieVerticalNode,chars) + (fChars.length()*sizeof(uint16_t)); |
-+ if(fValue > 0){ |
-+ estimatedSize += sizeof(uint16_t); |
-+ } |
-+ |
-+ if(fEqual->fNodeID > 0xFFFF){ |
-+ estimatedSize += sizeof(uint16_t); |
-+ } |
-+ return estimatedSize; |
- } |
- |
- virtual void write(uint8_t *bytes, uint32_t &offset, const UVector32 &translate) { |
- CompactTrieVerticalNode *node = (CompactTrieVerticalNode *)(bytes+offset); |
-+ fEqualOverflows = (translate.elementAti(fEqual->fNodeID) > 0xFFFF); |
- BuildCompactTrieNode::write(bytes, offset, translate); |
- node->equal = translate.elementAti(fEqual->fNodeID); |
- offset += sizeof(node->equal); |
- #ifdef DEBUG_TRIE_DICT |
-- if (node->equal == 0) { |
-+ if ((node->equal == 0) && !fEqualOverflows) { |
- fprintf(stderr, "ERROR: vertical link, logical node %d maps to physical node zero\n", |
- fEqual->fNodeID); |
- } |
- #endif |
- fChars.extract(0, fChars.length(), (UChar *)node->chars); |
-- offset += sizeof(uint16_t)*fChars.length(); |
-+ offset += sizeof(UChar)*fChars.length(); |
-+ |
-+ // append 16 bits of to end for equal node if fEqualOverflows |
-+ if (fEqualOverflows) { |
-+ *((uint16_t *)(bytes+offset)) = (translate.elementAti(fEqual->fNodeID) >> 16); |
-+ offset += sizeof(uint16_t); |
-+ } |
-+ |
-+ BuildCompactTrieNode::writeValue(bytes, offset); |
- } |
- |
- void addChar(UChar ch) { |
-@@ -784,60 +1161,85 @@ |
- void setLink(BuildCompactTrieNode *node) { |
- fEqual = node; |
- } |
-+ |
- }; |
- |
- // Forward declaration |
- static void walkHorizontal(const TernaryNode *node, |
- BuildCompactTrieHorizontalNode *building, |
- UStack &nodes, |
-- UErrorCode &status); |
-+ UErrorCode &status, |
-+ Hashtable *values); |
- |
--// Convert one node. Uses recursion. |
-+// Convert one TernaryNode into a BuildCompactTrieNode. Uses recursion. |
- |
- static BuildCompactTrieNode * |
--compactOneNode(const TernaryNode *node, UBool parentEndsWord, UStack &nodes, UErrorCode &status) { |
-+compactOneNode(const TernaryNode *node, UBool parentEndsWord, UStack &nodes, |
-+ UErrorCode &status, Hashtable *values = NULL, uint16_t parentValue = 0) { |
- if (U_FAILURE(status)) { |
- return NULL; |
- } |
- BuildCompactTrieNode *result = NULL; |
- UBool horizontal = (node->low != NULL || node->high != NULL); |
- if (horizontal) { |
-- BuildCompactTrieHorizontalNode *hResult = |
-- new BuildCompactTrieHorizontalNode(parentEndsWord, nodes, status); |
-+ BuildCompactTrieHorizontalNode *hResult; |
-+ if(values != NULL){ |
-+ hResult = new BuildCompactTrieHorizontalNode(parentEndsWord, nodes, status, parentValue); |
-+ } else { |
-+ hResult = new BuildCompactTrieHorizontalNode(parentEndsWord, nodes, status); |
-+ } |
-+ |
- if (hResult == NULL) { |
- status = U_MEMORY_ALLOCATION_ERROR; |
- return NULL; |
- } |
- if (U_SUCCESS(status)) { |
-- walkHorizontal(node, hResult, nodes, status); |
-+ walkHorizontal(node, hResult, nodes, status, values); |
- result = hResult; |
- } |
- } |
- else { |
-- BuildCompactTrieVerticalNode *vResult = |
-- new BuildCompactTrieVerticalNode(parentEndsWord, nodes, status); |
-+ BuildCompactTrieVerticalNode *vResult; |
-+ if(values != NULL){ |
-+ vResult = new BuildCompactTrieVerticalNode(parentEndsWord, nodes, status, parentValue); |
-+ } else { |
-+ vResult = new BuildCompactTrieVerticalNode(parentEndsWord, nodes, status); |
-+ } |
-+ |
- if (vResult == NULL) { |
- status = U_MEMORY_ALLOCATION_ERROR; |
-+ return NULL; |
- } |
- else if (U_SUCCESS(status)) { |
-- UBool endsWord = FALSE; |
-+ uint16_t value = 0; |
-+ UBool endsWord = FALSE; |
- // Take up nodes until we end a word, or hit a node with < or > links |
- do { |
- vResult->addChar(node->ch); |
-- endsWord = (node->flags & kEndsWord) != 0; |
-+ value = node->flags; |
-+ endsWord = value > 0; |
- node = node->equal; |
- } |
- while(node != NULL && !endsWord && node->low == NULL && node->high == NULL); |
-+ |
- if (node == NULL) { |
- if (!endsWord) { |
- status = U_ILLEGAL_ARGUMENT_ERROR; // Corrupt input trie |
- } |
-- else { |
-+ else if(values != NULL){ |
-+ UnicodeString key(value); //store value as a single-char UnicodeString |
-+ BuildCompactTrieValueNode *link = (BuildCompactTrieValueNode *) values->get(key); |
-+ if(link == NULL){ |
-+ link = new BuildCompactTrieValueNode(nodes, status, value); //take out nodes? |
-+ values->put(key, link, status); |
-+ } |
-+ vResult->setLink(link); |
-+ } else { |
- vResult->setLink((BuildCompactTrieNode *)nodes[1]); |
- } |
- } |
- else { |
-- vResult->setLink(compactOneNode(node, endsWord, nodes, status)); |
-+ vResult->setLink(compactOneNode(node, endsWord, nodes, status, values, value)); |
- } |
- result = vResult; |
- } |
-@@ -849,19 +1251,28 @@ |
- // Uses recursion. |
- |
- static void walkHorizontal(const TernaryNode *node, |
-- BuildCompactTrieHorizontalNode *building, |
-- UStack &nodes, |
-- UErrorCode &status) { |
-+ BuildCompactTrieHorizontalNode *building, |
-+ UStack &nodes, |
-+ UErrorCode &status, Hashtable *values = NULL) { |
- while (U_SUCCESS(status) && node != NULL) { |
- if (node->low != NULL) { |
-- walkHorizontal(node->low, building, nodes, status); |
-+ walkHorizontal(node->low, building, nodes, status, values); |
- } |
- BuildCompactTrieNode *link = NULL; |
- if (node->equal != NULL) { |
-- link = compactOneNode(node->equal, (node->flags & kEndsWord) != 0, nodes, status); |
-+ link = compactOneNode(node->equal, node->flags > 0, nodes, status, values, node->flags); |
- } |
-- else if (node->flags & kEndsWord) { |
-- link = (BuildCompactTrieNode *)nodes[1]; |
-+ else if (node->flags > 0) { |
-+ if(values != NULL) { |
-+ UnicodeString key(node->flags); //store value as a single-char UnicodeString |
-+ link = (BuildCompactTrieValueNode *) values->get(key); |
-+ if(link == NULL) { |
-+ link = new BuildCompactTrieValueNode(nodes, status, node->flags); //take out nodes? |
-+ values->put(key, link, status); |
-+ } |
-+ } else { |
-+ link = (BuildCompactTrieNode *)nodes[1]; |
-+ } |
- } |
- if (U_SUCCESS(status) && link != NULL) { |
- building->addNode(node->ch, link, status); |
-@@ -881,13 +1292,15 @@ |
- _sortBuildNodes(const void * /*context*/, const void *voidl, const void *voidr) { |
- BuildCompactTrieNode *left = *(BuildCompactTrieNode **)voidl; |
- BuildCompactTrieNode *right = *(BuildCompactTrieNode **)voidr; |
-+ |
- // Check for comparing a node to itself, to avoid spurious duplicates |
- if (left == right) { |
- return 0; |
- } |
-+ |
- // Most significant is type of node. Can never coalesce. |
-- if (left->fVertical != right->fVertical) { |
-- return left->fVertical - right->fVertical; |
-+ if (left->fNodeType != right->fNodeType) { |
-+ return left->fNodeType - right->fNodeType; |
- } |
- // Next, the "parent ends word" flag. If that differs, we cannot coalesce. |
- if (left->fParentEndsWord != right->fParentEndsWord) { |
-@@ -898,12 +1311,19 @@ |
- if (result != 0) { |
- return result; |
- } |
-+ |
-+ // If the node value differs, we should not coalesce. |
-+ // If values aren't stored, all fValues should be 0. |
-+ if (left->fValue != right->fValue) { |
-+ return left->fValue - right->fValue; |
-+ } |
-+ |
- // We know they're both the same node type, so branch for the two cases. |
-- if (left->fVertical) { |
-+ if (left->fNodeType == kVerticalType) { |
- result = ((BuildCompactTrieVerticalNode *)left)->fEqual->fNodeID |
-- - ((BuildCompactTrieVerticalNode *)right)->fEqual->fNodeID; |
-+ - ((BuildCompactTrieVerticalNode *)right)->fEqual->fNodeID; |
- } |
-- else { |
-+ else if(left->fChars.length() > 0 && right->fChars.length() > 0){ |
- // We need to compare the links vectors. They should be the |
- // same size because the strings were equal. |
- // We compare the node IDs instead of the pointers, to handle |
-@@ -914,9 +1334,10 @@ |
- int32_t count = hleft->fLinks.size(); |
- for (int32_t i = 0; i < count && result == 0; ++i) { |
- result = ((BuildCompactTrieNode *)(hleft->fLinks[i]))->fNodeID - |
-- ((BuildCompactTrieNode *)(hright->fLinks[i]))->fNodeID; |
-+ ((BuildCompactTrieNode *)(hright->fLinks[i]))->fNodeID; |
- } |
- } |
-+ |
- // If they are equal to each other, mark them (speeds coalescing) |
- if (result == 0) { |
- left->fHasDuplicate = TRUE; |
-@@ -1031,20 +1452,25 @@ |
- // Add node 0, used as the NULL pointer/sentinel. |
- nodes.addElement((int32_t)0, status); |
- |
-+ Hashtable *values = NULL; // Index of (unique) values |
-+ if (dict.fValued) { |
-+ values = new Hashtable(status); |
-+ } |
-+ |
- // Start by creating the special empty node we use to indicate that the parent |
- // terminates a word. This must be node 1, because the builder assumes |
-- // that. |
-+ // that. This node will never be used for tries storing numerical values. |
- if (U_FAILURE(status)) { |
- return NULL; |
- } |
-- BuildCompactTrieNode *terminal = new BuildCompactTrieNode(TRUE, FALSE, nodes, status); |
-+ BuildCompactTrieNode *terminal = new BuildCompactTrieNode(TRUE, kHorizontalType, nodes, status); |
- if (terminal == NULL) { |
- status = U_MEMORY_ALLOCATION_ERROR; |
- } |
- |
- // This call does all the work of building the new trie structure. The root |
-- // will be node 2. |
-- BuildCompactTrieNode *root = compactOneNode(dict.fTrie, FALSE, nodes, status); |
-+ // will have node ID 2 before writing to memory. |
-+ BuildCompactTrieNode *root = compactOneNode(dict.fTrie, FALSE, nodes, status, values); |
- #ifdef DEBUG_TRIE_DICT |
- (void) ::times(&timing); |
- fprintf(stderr, "Compact trie built, %d nodes, time user %f system %f\n", |
-@@ -1077,21 +1503,37 @@ |
- return NULL; |
- } |
- |
-+ //map terminal value nodes |
-+ int valueCount = 0; |
-+ UVector valueNodes(status); |
-+ if(values != NULL) { |
-+ valueCount = values->count(); //number of unique terminal value nodes |
-+ } |
-+ |
-+ // map non-terminal nodes |
-+ int valuePos = 1;//, nodePos = valueCount + valuePos; |
-+ nodeCount = valueCount + valuePos; |
- for (i = 1; i < count; ++i) { |
- node = (BuildCompactTrieNode *)nodes[i]; |
- if (node->fNodeID == i) { |
- // Only one node out of each duplicate set is used |
-- if (i >= translate.size()) { |
-+ if (node->fNodeID >= translate.size()) { |
- // Logically extend the mapping table |
-- translate.setSize(i+1); |
-+ translate.setSize(i + 1); |
-+ } |
-+ //translate.setElementAt(object, index)! |
-+ if(node->fNodeType == kValueType) { |
-+ valueNodes.addElement(node, status); |
-+ translate.setElementAt(valuePos++, i); |
-+ } else { |
-+ translate.setElementAt(nodeCount++, i); |
- } |
-- translate.setElementAt(nodeCount++, i); |
- totalSize += node->size(); |
- } |
- } |
-- |
-- // Check for overflowing 16 bits worth of nodes. |
-- if (nodeCount > 0x10000) { |
-+ |
-+ // Check for overflowing 20 bits worth of nodes. |
-+ if (nodeCount > 0x100000) { |
- status = U_ILLEGAL_ARGUMENT_ERROR; |
- return NULL; |
- } |
-@@ -1111,9 +1553,14 @@ |
- status = U_MEMORY_ALLOCATION_ERROR; |
- return NULL; |
- } |
-- |
-+ |
- CompactTrieHeader *header = (CompactTrieHeader *)bytes; |
-- header->size = totalSize; |
-+ //header->size = totalSize; |
-+ if(dict.fValued){ |
-+ header->magic = COMPACT_TRIE_MAGIC_3; |
-+ } else { |
-+ header->magic = COMPACT_TRIE_MAGIC_2; |
-+ } |
- header->nodeCount = nodeCount; |
- header->offsets[0] = 0; // Sentinel |
- header->root = translate.elementAti(root->fNodeID); |
-@@ -1123,23 +1570,40 @@ |
- } |
- #endif |
- uint32_t offset = offsetof(CompactTrieHeader,offsets)+(nodeCount*sizeof(uint32_t)); |
-- nodeCount = 1; |
-+ nodeCount = valueCount + 1; |
-+ |
-+ // Write terminal value nodes to memory |
-+ for (i=0; i < valueNodes.size(); i++) { |
-+ //header->offsets[i + 1] = offset; |
-+ uint32_t tmpOffset = 0; |
-+ node = (BuildCompactTrieNode *) valueNodes.elementAt(i); |
-+ //header->offsets[i + 1] = (uint32_t)node->fValue; |
-+ node->write((uint8_t *)&header->offsets[i+1], tmpOffset, translate); |
-+ } |
-+ |
- // Now write the data |
- for (i = 1; i < count; ++i) { |
- node = (BuildCompactTrieNode *)nodes[i]; |
-- if (node->fNodeID == i) { |
-+ if (node->fNodeID == i && node->fNodeType != kValueType) { |
- header->offsets[nodeCount++] = offset; |
- node->write(bytes, offset, translate); |
- } |
- } |
-+ |
-+ //free all extra space |
-+ uprv_realloc(bytes, offset); |
-+ header->size = offset; |
-+ |
- #ifdef DEBUG_TRIE_DICT |
-+ fprintf(stdout, "Space freed: %d\n", totalSize-offset); |
-+ |
- (void) ::times(&timing); |
- fprintf(stderr, "Trie built, time user %f system %f\n", |
- (double)(timing.tms_utime-previous.tms_utime)/CLK_TCK, |
- (double)(timing.tms_stime-previous.tms_stime)/CLK_TCK); |
- previous = timing; |
- fprintf(stderr, "Final offset is %d\n", offset); |
-- |
-+ |
- // Collect statistics on node types and sizes |
- int hCount = 0; |
- int vCount = 0; |
-@@ -1148,68 +1612,85 @@ |
- size_t hItemCount = 0; |
- size_t vItemCount = 0; |
- uint32_t previousOff = offset; |
-- for (uint16_t nodeIdx = nodeCount-1; nodeIdx >= 2; --nodeIdx) { |
-+ uint32_t numOverflow = 0; |
-+ uint32_t valueSpace = 0; |
-+ for (uint32_t nodeIdx = nodeCount-1; nodeIdx >= 2; --nodeIdx) { |
- const CompactTrieNode *node = getCompactNode(header, nodeIdx); |
-- if (node->flagscount & kVerticalNode) { |
-+ int itemCount; |
-+ if(nodeIdx == header->root) |
-+ itemCount = node->flagscount & kRootCountMask; |
-+ else |
-+ itemCount = getCount(node); |
-+ if(node->flagscount & kEqualOverflows){ |
-+ numOverflow++; |
-+ } |
-+ if (node->flagscount & kVerticalNode && nodeIdx != header->root) { |
- vCount += 1; |
-- vItemCount += (node->flagscount & kCountMask); |
-+ vItemCount += itemCount; |
- vSize += previousOff-header->offsets[nodeIdx]; |
- } |
- else { |
- hCount += 1; |
-- hItemCount += (node->flagscount & kCountMask); |
-- hSize += previousOff-header->offsets[nodeIdx]; |
-+ hItemCount += itemCount; |
-+ if(nodeIdx >= header->root) { |
-+ hSize += previousOff-header->offsets[nodeIdx]; |
-+ } |
- } |
-+ |
-+ if(header->magic == COMPACT_TRIE_MAGIC_3 && node->flagscount & kParentEndsWord) |
-+ valueSpace += sizeof(uint16_t); |
- previousOff = header->offsets[nodeIdx]; |
- } |
- fprintf(stderr, "Horizontal nodes: %d total, average %f bytes with %f items\n", hCount, |
- (double)hSize/hCount, (double)hItemCount/hCount); |
- fprintf(stderr, "Vertical nodes: %d total, average %f bytes with %f items\n", vCount, |
- (double)vSize/vCount, (double)vItemCount/vCount); |
-+ fprintf(stderr, "Number of nodes with overflowing nodeIDs: %d \n", numOverflow); |
-+ fprintf(stderr, "Space taken up by values: %d \n", valueSpace); |
- #endif |
- |
- if (U_FAILURE(status)) { |
- uprv_free(bytes); |
- header = NULL; |
- } |
-- else { |
-- header->magic = COMPACT_TRIE_MAGIC_1; |
-- } |
- return header; |
- } |
- |
- // Forward declaration |
- static TernaryNode * |
--unpackOneNode( const CompactTrieHeader *header, const CompactTrieNode *node, UErrorCode &status ); |
-- |
-+unpackOneNode( const CompactTrieInfo *info, const CompactTrieNode *node, UErrorCode &status ); |
- |
- // Convert a horizontal node (or subarray thereof) into a ternary subtrie |
- static TernaryNode * |
--unpackHorizontalArray( const CompactTrieHeader *header, const CompactTrieHorizontalEntry *array, |
-- int low, int high, UErrorCode &status ) { |
-+unpackHorizontalArray( const CompactTrieInfo *info, const CompactTrieHorizontalNode *hnode, |
-+ int low, int high, int nodeCount, UErrorCode &status) { |
- if (U_FAILURE(status) || low > high) { |
- return NULL; |
- } |
- int middle = (low+high)/2; |
-- TernaryNode *result = new TernaryNode(array[middle].ch); |
-+ TernaryNode *result = new TernaryNode(hnode->entries[middle].ch); |
- if (result == NULL) { |
- status = U_MEMORY_ALLOCATION_ERROR; |
- return NULL; |
- } |
-- const CompactTrieNode *equal = getCompactNode(header, array[middle].equal); |
-+ const CompactTrieNode *equal = getCompactNode(info, calcEqualLink(hnode, middle, nodeCount)); |
- if (equal->flagscount & kParentEndsWord) { |
-- result->flags |= kEndsWord; |
-+ if(info->magic == COMPACT_TRIE_MAGIC_3){ |
-+ result->flags = getValue(equal); |
-+ }else{ |
-+ result->flags |= kEndsWord; |
-+ } |
- } |
-- result->low = unpackHorizontalArray(header, array, low, middle-1, status); |
-- result->high = unpackHorizontalArray(header, array, middle+1, high, status); |
-- result->equal = unpackOneNode(header, equal, status); |
-+ result->low = unpackHorizontalArray(info, hnode, low, middle-1, nodeCount, status); |
-+ result->high = unpackHorizontalArray(info, hnode, middle+1, high, nodeCount, status); |
-+ result->equal = unpackOneNode(info, equal, status); |
- return result; |
- } |
- |
- // Convert one compact trie node into a ternary subtrie |
- static TernaryNode * |
--unpackOneNode( const CompactTrieHeader *header, const CompactTrieNode *node, UErrorCode &status ) { |
-- int nodeCount = (node->flagscount & kCountMask); |
-+unpackOneNode( const CompactTrieInfo *info, const CompactTrieNode *node, UErrorCode &status ) { |
-+ int nodeCount = getCount(node); |
- if (nodeCount == 0 || U_FAILURE(status)) { |
- // Failure, or terminal node |
- return NULL; |
-@@ -1234,29 +1715,41 @@ |
- previous = latest; |
- } |
- if (latest != NULL) { |
-- const CompactTrieNode *equal = getCompactNode(header, vnode->equal); |
-+ const CompactTrieNode *equal = getCompactNode(info, calcEqualLink(vnode)); |
- if (equal->flagscount & kParentEndsWord) { |
-- latest->flags |= kEndsWord; |
-+ if(info->magic == COMPACT_TRIE_MAGIC_3){ |
-+ latest->flags = getValue(equal); |
-+ } else { |
-+ latest->flags |= kEndsWord; |
-+ } |
- } |
-- latest->equal = unpackOneNode(header, equal, status); |
-+ latest->equal = unpackOneNode(info, equal, status); |
- } |
- return head; |
- } |
- else { |
- // Horizontal node |
- const CompactTrieHorizontalNode *hnode = (const CompactTrieHorizontalNode *)node; |
-- return unpackHorizontalArray(header, &hnode->entries[0], 0, nodeCount-1, status); |
-+ return unpackHorizontalArray(info, hnode, 0, nodeCount-1, nodeCount, status); |
- } |
- } |
- |
-+// returns a MutableTrieDictionary generated from the CompactTrieDictionary |
- MutableTrieDictionary * |
- CompactTrieDictionary::cloneMutable( UErrorCode &status ) const { |
-- MutableTrieDictionary *result = new MutableTrieDictionary( status ); |
-+ MutableTrieDictionary *result = new MutableTrieDictionary( status, fInfo->magic == COMPACT_TRIE_MAGIC_3 ); |
- if (result == NULL) { |
- status = U_MEMORY_ALLOCATION_ERROR; |
- return NULL; |
- } |
-- TernaryNode *root = unpackOneNode(fData, getCompactNode(fData, fData->root), status); |
-+ // treat root node as special case: don't call unpackOneNode() or unpackHorizontalArray() directly |
-+ // because only kEqualOverflows flag should be checked in root's flagscount |
-+ const CompactTrieHorizontalNode *hnode = (const CompactTrieHorizontalNode *) |
-+ getCompactNode(fInfo, fInfo->root); |
-+ uint16_t nodeCount = hnode->flagscount & kRootCountMask; |
-+ TernaryNode *root = unpackHorizontalArray(fInfo, hnode, 0, nodeCount-1, |
-+ nodeCount, status); |
-+ |
- if (U_FAILURE(status)) { |
- delete root; // Clean up |
- delete result; |
-@@ -1270,8 +1763,8 @@ |
- |
- U_CAPI int32_t U_EXPORT2 |
- triedict_swap(const UDataSwapper *ds, const void *inData, int32_t length, void *outData, |
-- UErrorCode *status) { |
-- |
-+ UErrorCode *status) { |
-+ |
- if (status == NULL || U_FAILURE(*status)) { |
- return 0; |
- } |
-@@ -1286,14 +1779,14 @@ |
- // |
- const UDataInfo *pInfo = (const UDataInfo *)((const uint8_t *)inData+4); |
- if(!( pInfo->dataFormat[0]==0x54 && /* dataFormat="TrDc" */ |
-- pInfo->dataFormat[1]==0x72 && |
-- pInfo->dataFormat[2]==0x44 && |
-- pInfo->dataFormat[3]==0x63 && |
-- pInfo->formatVersion[0]==1 )) { |
-+ pInfo->dataFormat[1]==0x72 && |
-+ pInfo->dataFormat[2]==0x44 && |
-+ pInfo->dataFormat[3]==0x63 && |
-+ pInfo->formatVersion[0]==1 )) { |
- udata_printError(ds, "triedict_swap(): data format %02x.%02x.%02x.%02x (format version %02x) is not recognized\n", |
-- pInfo->dataFormat[0], pInfo->dataFormat[1], |
-- pInfo->dataFormat[2], pInfo->dataFormat[3], |
-- pInfo->formatVersion[0]); |
-+ pInfo->dataFormat[0], pInfo->dataFormat[1], |
-+ pInfo->dataFormat[2], pInfo->dataFormat[3], |
-+ pInfo->formatVersion[0]); |
- *status=U_UNSUPPORTED_ERROR; |
- return 0; |
- } |
-@@ -1311,8 +1804,10 @@ |
- // |
- const uint8_t *inBytes =(const uint8_t *)inData+headerSize; |
- const CompactTrieHeader *header = (const CompactTrieHeader *)inBytes; |
-- if (ds->readUInt32(header->magic) != COMPACT_TRIE_MAGIC_1 |
-- || ds->readUInt32(header->size) < sizeof(CompactTrieHeader)) |
-+ uint32_t magic = ds->readUInt32(header->magic); |
-+ if (magic != COMPACT_TRIE_MAGIC_1 && magic != COMPACT_TRIE_MAGIC_2 && magic != COMPACT_TRIE_MAGIC_3 |
-+ || magic == COMPACT_TRIE_MAGIC_1 && ds->readUInt32(header->size) < sizeof(CompactTrieHeaderV1) |
-+ || magic != COMPACT_TRIE_MAGIC_1 && ds->readUInt32(header->size) < sizeof(CompactTrieHeader)) |
- { |
- udata_printError(ds, "triedict_swap(): CompactTrieHeader is invalid.\n"); |
- *status=U_UNSUPPORTED_ERROR; |
-@@ -1333,10 +1828,10 @@ |
- // |
- if (length < sizeWithUData) { |
- udata_printError(ds, "triedict_swap(): too few bytes (%d after ICU Data header) for trie data.\n", |
-- totalSize); |
-+ totalSize); |
- *status=U_INDEX_OUTOFBOUNDS_ERROR; |
- return 0; |
-- } |
-+ } |
- |
- // |
- // Swap the Data. Do the data itself first, then the CompactTrieHeader, because |
-@@ -1355,20 +1850,38 @@ |
- } |
- |
- // We need to loop through all the nodes in the offset table, and swap each one. |
-- uint16_t nodeCount = ds->readUInt16(header->nodeCount); |
-+ uint32_t nodeCount, rootId; |
-+ if(header->magic == COMPACT_TRIE_MAGIC_1) { |
-+ nodeCount = ds->readUInt16(((CompactTrieHeaderV1 *)header)->nodeCount); |
-+ rootId = ds->readUInt16(((CompactTrieHeaderV1 *)header)->root); |
-+ } else { |
-+ nodeCount = ds->readUInt32(header->nodeCount); |
-+ rootId = ds->readUInt32(header->root); |
-+ } |
-+ |
- // Skip node 0, which should always be 0. |
-- for (int i = 1; i < nodeCount; ++i) { |
-+ for (uint32_t i = 1; i < nodeCount; ++i) { |
- uint32_t nodeOff = ds->readUInt32(header->offsets[i]); |
- const CompactTrieNode *inNode = (const CompactTrieNode *)(inBytes + nodeOff); |
- CompactTrieNode *outNode = (CompactTrieNode *)(outBytes + nodeOff); |
- uint16_t flagscount = ds->readUInt16(inNode->flagscount); |
-- uint16_t itemCount = flagscount & kCountMask; |
-+ uint16_t itemCount = getCount(inNode); |
-+ //uint16_t itemCount = flagscount & kCountMask; |
- ds->writeUInt16(&outNode->flagscount, flagscount); |
- if (itemCount > 0) { |
-- if (flagscount & kVerticalNode) { |
-+ uint16_t overflow = 0; //number of extra uint16_ts needed to be swapped |
-+ if (flagscount & kVerticalNode && i != rootId) { |
-+ if(flagscount & kEqualOverflows){ |
-+ // include overflow bits |
-+ overflow += 1; |
-+ } |
-+ if (header->magic == COMPACT_TRIE_MAGIC_3 && flagscount & kEndsParentWord) { |
-+ //include values |
-+ overflow += 1; |
-+ } |
- ds->swapArray16(ds, inBytes+nodeOff+offsetof(CompactTrieVerticalNode,chars), |
-- itemCount*sizeof(uint16_t), |
-- outBytes+nodeOff+offsetof(CompactTrieVerticalNode,chars), status); |
-+ (itemCount + overflow)*sizeof(uint16_t), |
-+ outBytes+nodeOff+offsetof(CompactTrieVerticalNode,chars), status); |
- uint16_t equal = ds->readUInt16(inBytes+nodeOff+offsetof(CompactTrieVerticalNode,equal); |
- ds->writeUInt16(outBytes+nodeOff+offsetof(CompactTrieVerticalNode,equal)); |
- } |
-@@ -1381,26 +1894,62 @@ |
- word = ds->readUInt16(inHNode->entries[j].equal); |
- ds->writeUInt16(&outHNode->entries[j].equal, word); |
- } |
-+ |
-+ // swap overflow/value information |
-+ if(flagscount & kEqualOverflows){ |
-+ overflow += (itemCount + 3) / 4; |
-+ } |
-+ |
-+ if (header->magic == COMPACT_TRIE_MAGIC_3 && i != rootId && flagscount & kEndsParentWord) { |
-+ //include values |
-+ overflow += 1; |
-+ } |
-+ |
-+ uint16_t *inOverflow = (uint16_t *) &inHNode->entries[itemCount]; |
-+ uint16_t *outOverflow = (uint16_t *) &outHNode->entries[itemCount]; |
-+ for(int j = 0; j<overflow; j++){ |
-+ uint16_t extraInfo = ds->readUInt16(*inOverflow); |
-+ ds->writeUInt16(outOverflow, extraInfo); |
-+ |
-+ inOverflow++; |
-+ outOverflow++; |
-+ } |
- } |
- } |
- } |
- #endif |
- |
-- // All the data in all the nodes consist of 16 bit items. Swap them all at once. |
-- uint16_t nodeCount = ds->readUInt16(header->nodeCount); |
-- uint32_t nodesOff = offsetof(CompactTrieHeader,offsets)+((uint32_t)nodeCount*sizeof(uint32_t)); |
-- ds->swapArray16(ds, inBytes+nodesOff, totalSize-nodesOff, outBytes+nodesOff, status); |
-- |
- // Swap the header |
- ds->writeUInt32(&outputHeader->size, totalSize); |
-- uint32_t magic = ds->readUInt32(header->magic); |
- ds->writeUInt32(&outputHeader->magic, magic); |
-- ds->writeUInt16(&outputHeader->nodeCount, nodeCount); |
-- uint16_t root = ds->readUInt16(header->root); |
-- ds->writeUInt16(&outputHeader->root, root); |
-- ds->swapArray32(ds, inBytes+offsetof(CompactTrieHeader,offsets), |
-- sizeof(uint32_t)*(int32_t)nodeCount, |
-- outBytes+offsetof(CompactTrieHeader,offsets), status); |
-+ |
-+ uint32_t nodeCount; |
-+ uint32_t offsetPos; |
-+ if (header->magic == COMPACT_TRIE_MAGIC_1) { |
-+ CompactTrieHeaderV1 *headerV1 = (CompactTrieHeaderV1 *)header; |
-+ CompactTrieHeaderV1 *outputHeaderV1 = (CompactTrieHeaderV1 *)outputHeader; |
-+ |
-+ nodeCount = ds->readUInt16(headerV1->nodeCount); |
-+ ds->writeUInt16(&outputHeaderV1->nodeCount, nodeCount); |
-+ uint16_t root = ds->readUInt16(headerV1->root); |
-+ ds->writeUInt16(&outputHeaderV1->root, root); |
-+ offsetPos = offsetof(CompactTrieHeaderV1,offsets); |
-+ } else { |
-+ nodeCount = ds->readUInt32(header->nodeCount); |
-+ ds->writeUInt32(&outputHeader->nodeCount, nodeCount); |
-+ uint32_t root = ds->readUInt32(header->root); |
-+ ds->writeUInt32(&outputHeader->root, root); |
-+ offsetPos = offsetof(CompactTrieHeader,offsets); |
-+ } |
-+ |
-+ // All the data in all the nodes consist of 16 bit items. Swap them all at once. |
-+ uint32_t nodesOff = offsetPos+((uint32_t)nodeCount*sizeof(uint32_t)); |
-+ ds->swapArray16(ds, inBytes+nodesOff, totalSize-nodesOff, outBytes+nodesOff, status); |
-+ |
-+ //swap offsets |
-+ ds->swapArray32(ds, inBytes+offsetPos, |
-+ sizeof(uint32_t)*(uint32_t)nodeCount, |
-+ outBytes+offsetPos, status); |
- |
- return sizeWithUData; |
- } |
---- source/common/triedict.h 2006-06-06 15:38:49.000000000 -0700 |
-+++ source/common/triedict.h 2011-01-21 14:12:45.496927000 -0800 |
-@@ -47,7 +47,6 @@ |
- U_NAMESPACE_BEGIN |
- |
- class StringEnumeration; |
--struct CompactTrieHeader; |
- |
- /******************************************************************* |
- * TrieWordDictionary |
-@@ -72,23 +71,29 @@ |
- */ |
- virtual ~TrieWordDictionary(); |
- |
-+ /** |
-+ * <p>Returns true if the dictionary contains values associated with each word.</p> |
-+ */ |
-+ virtual UBool getValued() const = 0; |
-+ |
- /** |
- * <p>Find dictionary words that match the text.</p> |
- * |
- * @param text A UText representing the text. The |
- * iterator is left after the longest prefix match in the dictionary. |
-- * @param start The current position in text. |
- * @param maxLength The maximum number of code units to match. |
- * @param lengths An array that is filled with the lengths of words that matched. |
- * @param count Filled with the number of elements output in lengths. |
- * @param limit The size of the lengths array; this limits the number of words output. |
-+ * @param values An array that is filled with the values associated with the matched words. |
- * @return The number of characters in text that were matched. |
- */ |
- virtual int32_t matches( UText *text, |
- int32_t maxLength, |
- int32_t *lengths, |
- int &count, |
-- int limit ) const = 0; |
-+ int limit, |
-+ uint16_t *values = NULL) const = 0; |
- |
- /** |
- * <p>Return a StringEnumeration for iterating all the words in the dictionary.</p> |
-@@ -128,6 +133,12 @@ |
- |
- UText *fIter; |
- |
-+ /** |
-+ * A UText for internal use |
-+ * @internal |
-+ */ |
-+ UBool fValued; |
-+ |
- friend class CompactTrieDictionary; // For fast conversion |
- |
- public: |
-@@ -138,14 +149,29 @@ |
- * @param median A UChar around which to balance the trie. Ideally, it should |
- * begin at least one word that is near the median of the set in the dictionary |
- * @param status A status code recording the success of the call. |
-+ * @param containsValue True if the dictionary stores values associated with each word. |
- */ |
-- MutableTrieDictionary( UChar median, UErrorCode &status ); |
-+ MutableTrieDictionary( UChar median, UErrorCode &status, UBool containsValue = FALSE ); |
- |
- /** |
- * <p>Virtual destructor.</p> |
- */ |
- virtual ~MutableTrieDictionary(); |
- |
-+ /** |
-+ * Indicate whether the MutableTrieDictionary stores values associated with each word |
-+ */ |
-+ void setValued(UBool valued){ |
-+ fValued = valued; |
-+ } |
-+ |
-+ /** |
-+ * <p>Returns true if the dictionary contains values associated with each word.</p> |
-+ */ |
-+ virtual UBool getValued() const { |
-+ return fValued; |
-+ } |
-+ |
- /** |
- * <p>Find dictionary words that match the text.</p> |
- * |
-@@ -155,13 +181,15 @@ |
- * @param lengths An array that is filled with the lengths of words that matched. |
- * @param count Filled with the number of elements output in lengths. |
- * @param limit The size of the lengths array; this limits the number of words output. |
-+ * @param values An array that is filled with the values associated with the matched words. |
- * @return The number of characters in text that were matched. |
- */ |
- virtual int32_t matches( UText *text, |
- int32_t maxLength, |
- int32_t *lengths, |
- int &count, |
-- int limit ) const; |
-+ int limit, |
-+ uint16_t *values = NULL) const; |
- |
- /** |
- * <p>Return a StringEnumeration for iterating all the words in the dictionary.</p> |
-@@ -173,15 +201,17 @@ |
- virtual StringEnumeration *openWords( UErrorCode &status ) const; |
- |
- /** |
-- * <p>Add one word to the dictionary.</p> |
-+ * <p>Add one word to the dictionary with an optional associated value.</p> |
- * |
- * @param word A UChar buffer containing the word. |
- * @param length The length of the word. |
-- * @param status The resultant status |
-+ * @param status The resultant status. |
-+ * @param value The nonzero value associated with this word. |
- */ |
- virtual void addWord( const UChar *word, |
- int32_t length, |
-- UErrorCode &status); |
-+ UErrorCode &status, |
-+ uint16_t value = 0); |
- |
- #if 0 |
- /** |
-@@ -203,8 +233,9 @@ |
- * @param lengths An array that is filled with the lengths of words that matched. |
- * @param count Filled with the number of elements output in lengths. |
- * @param limit The size of the lengths array; this limits the number of words output. |
-- * @param parent The parent of the current node |
-- * @param pMatched The returned parent node matched the input |
-+ * @param parent The parent of the current node. |
-+ * @param pMatched The returned parent node matched the input/ |
-+ * @param values An array that is filled with the values associated with the matched words. |
- * @return The number of characters in text that were matched. |
- */ |
- virtual int32_t search( UText *text, |
-@@ -213,40 +244,46 @@ |
- int &count, |
- int limit, |
- TernaryNode *&parent, |
-- UBool &pMatched ) const; |
-+ UBool &pMatched, |
-+ uint16_t *values = NULL) const; |
- |
- private: |
- /** |
- * <p>Private constructor. The root node it not allocated.</p> |
- * |
- * @param status A status code recording the success of the call. |
-+ * @param containsValues True if the dictionary will store a value associated |
-+ * with each word added. |
- */ |
-- MutableTrieDictionary( UErrorCode &status ); |
-+ MutableTrieDictionary( UErrorCode &status, UBool containsValues = false ); |
- }; |
- |
- /******************************************************************* |
- * CompactTrieDictionary |
- */ |
- |
-+//forward declarations |
-+struct CompactTrieHeader; |
-+struct CompactTrieInfo; |
-+ |
- /** |
- * <p>CompactTrieDictionary is a TrieWordDictionary that has been compacted |
- * to save space.</p> |
- */ |
- class U_COMMON_API CompactTrieDictionary : public TrieWordDictionary { |
- private: |
-- /** |
-- * The root node of the trie |
-- */ |
-+ /** |
-+ * The header of the CompactTrieDictionary which contains all info |
-+ */ |
- |
-- const CompactTrieHeader *fData; |
-- |
-- /** |
-- * A UBool indicating whether or not we own the fData. |
-- */ |
-+ CompactTrieInfo *fInfo; |
- |
-+ /** |
-+ * A UBool indicating whether or not we own the fData. |
-+ */ |
- UBool fOwnData; |
- |
-- UDataMemory *fUData; |
-+ UDataMemory *fUData; |
- public: |
- /** |
- * <p>Construct a dictionary from a UDataMemory.</p> |
-@@ -277,6 +314,11 @@ |
- */ |
- virtual ~CompactTrieDictionary(); |
- |
-+ /** |
-+ * <p>Returns true if the dictionary contains values associated with each word.</p> |
-+ */ |
-+ virtual UBool getValued() const; |
-+ |
- /** |
- * <p>Find dictionary words that match the text.</p> |
- * |
-@@ -286,13 +328,15 @@ |
- * @param lengths An array that is filled with the lengths of words that matched. |
- * @param count Filled with the number of elements output in lengths. |
- * @param limit The size of the lengths array; this limits the number of words output. |
-+ * @param values An array that is filled with the values associated with the matched words. |
- * @return The number of characters in text that were matched. |
- */ |
- virtual int32_t matches( UText *text, |
-- int32_t rangeEnd, |
-+ int32_t maxLength, |
- int32_t *lengths, |
- int &count, |
-- int limit ) const; |
-+ int limit, |
-+ uint16_t *values = NULL) const; |
- |
- /** |
- * <p>Return a StringEnumeration for iterating all the words in the dictionary.</p> |
-@@ -311,7 +355,7 @@ |
- virtual uint32_t dataSize() const; |
- |
- /** |
-- * <p>Return a void * pointer to the compact data, platform-endian.</p> |
-+ * <p>Return a void * pointer to the (unmanaged) compact data, platform-endian.</p> |
- * |
- * @return The data for the compact dictionary, suitable for passing to the |
- * constructor. |
-@@ -342,5 +386,5 @@ |
- |
- U_NAMESPACE_END |
- |
-- /* TRIEDICT_H */ |
-+/* TRIEDICT_H */ |
- #endif |
---- source/data/Makefile.in 2010-10-29 13:21:33.000000000 -0700 |
-+++ source/data/Makefile.in 2011-01-26 16:24:24.856798000 -0800 |
-@@ -509,8 +520,9 @@ |
- #################################################### CTD |
- # CTD FILES |
- |
--$(BRKBLDDIR)/%.ctd: $(BRKSRCDIR)/%.txt $(TOOLBINDIR)/genctd$(TOOLEXEEXT) $(DAT_FILES) |
-- $(INVOKE) $(TOOLBINDIR)/genctd -c -i $(BUILDDIR) -o $@ $< |
-+# .ctd file now generated regardless of whether dictionary file exists |
-+$(BRKBLDDIR)/%.ctd: $(TOOLBINDIR)/genctd$(TOOLEXEEXT) $(DAT_FILES) |
-+ $(INVOKE) $(TOOLBINDIR)/genctd -c -i $(BUILDDIR) -o $@ $(BRKSRCDIR)/$(*F).txt |
- |
- #################################################### CFU |
- # CFU FILES |
---- source/data/brkitr/root.txt 2010-07-28 17:18:28.000000000 -0700 |
-+++ source/data/brkitr/root.txt 2011-01-21 14:12:45.653922000 -0800 |
-@@ -17,5 +17,8 @@ |
- } |
- dictionaries{ |
- Thai:process(dependency){"thaidict.ctd"} |
-+ Hani:process(dependency){"cjdict.ctd"} |
-+ Hira:process(dependency){"cjdict.ctd"} |
-+ Kata:process(dependency){"cjdict.ctd"} |
- } |
- } |
---- source/data/xml/brkitr/root.xml 2010-03-01 15:13:18.000000000 -0800 |
-+++ source/data/xml/brkitr/root.xml 2011-01-21 14:12:45.735922000 -0800 |
-@@ -25,6 +25,9 @@ |
- </icu:boundaries> |
- <icu:dictionaries> |
- <icu:dictionary type="Thai" icu:dependency="thaidict.ctd"/> |
-+ <icu:dictionary type="Hani" icu:dependency="cjdict.ctd"/> |
-+ <icu:dictionary type="Hira" icu:dependency="cjdict.ctd"/> |
-+ <icu:dictionary type="Kata" icu:dependency="cjdict.ctd"/> |
- </icu:dictionaries> |
- </icu:breakIteratorData> |
- </special> |
---- source/test/cintltst/creststn.c 2010-10-28 10:44:02.000000000 -0700 |
-+++ source/test/cintltst/creststn.c 2011-01-21 14:12:44.995020000 -0800 |
-@@ -2188,21 +2188,21 @@ |
- |
- |
- { |
-- UResourceBundle* ja = ures_open(U_ICUDATA_BRKITR,"ja", &status); |
-+ UResourceBundle* th = ures_open(U_ICUDATA_BRKITR,"th", &status); |
- const UChar *got = NULL, *exp=NULL; |
- int32_t gotLen = 0, expLen=0; |
-- ja = ures_getByKey(ja, "boundaries", ja, &status); |
-- exp = tres_getString(ja, -1, "word", &expLen, &status); |
-+ th = ures_getByKey(th, "boundaries", th, &status); |
-+ exp = tres_getString(th, -1, "grapheme", &expLen, &status); |
- |
- tb = ures_getByKey(aliasB, "boundaries", tb, &status); |
-- got = tres_getString(tb, -1, "word", &gotLen, &status); |
-+ got = tres_getString(tb, -1, "grapheme", &gotLen, &status); |
- |
- if(U_FAILURE(status)) { |
- log_err("%s trying to read str boundaries\n", u_errorName(status)); |
- } else if(gotLen != expLen || u_strncmp(exp, got, gotLen) != 0) { |
- log_err("Referencing alias didn't get the right data\n"); |
- } |
-- ures_close(ja); |
-+ ures_close(th); |
- status = U_ZERO_ERROR; |
- } |
- /* simple alias */ |
---- source/test/intltest/rbbiapts.cpp 2010-07-12 11:03:29.000000000 -0700 |
-+++ source/test/intltest/rbbiapts.cpp 2011-01-21 14:12:45.033014000 -0800 |
-@@ -156,9 +156,13 @@ |
- if(*a!=*b){ |
- errln("Failed: boilerplate method operator!= does not return correct results"); |
- } |
-- BreakIterator* c = BreakIterator::createWordInstance(Locale("ja"),status); |
-- if(a && c){ |
-- if(*c==*a){ |
-+ // Japanese word break iteratos is identical to root with |
-+ // a dictionary-based break iterator, but Thai character break iterator |
-+ // is still different from Root. |
-+ BreakIterator* c = BreakIterator::createCharacterInstance(Locale("ja"),status); |
-+ BreakIterator* d = BreakIterator::createCharacterInstance(Locale("th"),status); |
-+ if(c && d){ |
-+ if(*c==*d){ |
- errln("Failed: boilerplate method opertator== does not return correct results"); |
- } |
- }else{ |
-@@ -167,6 +171,7 @@ |
- delete a; |
- delete b; |
- delete c; |
-+ delete d; |
- } |
- |
- void RBBIAPITest::TestgetRules() |
-@@ -635,21 +640,21 @@ |
- // |
- void RBBIAPITest::TestRuleStatus() { |
- UChar str[30]; |
-- u_unescape("plain word 123.45 \\u9160\\u9161 \\u30a1\\u30a2 \\u3041\\u3094", |
-- // 012345678901234567 8 9 0 1 2 3 4 5 6 |
-- // Ideographic Katakana Hiragana |
-+ //no longer test Han or hiragana breaking here: ruleStatusVec would return nothing |
-+ // changed UBRK_WORD_KANA to UBRK_WORD_IDEO |
-+ u_unescape("plain word 123.45 \\u30a1\\u30a2 ", |
-+ // 012345678901234567 8 9 0 |
-+ // Katakana |
- str, 30); |
- UnicodeString testString1(str); |
-- int32_t bounds1[] = {0, 5, 6, 10, 11, 17, 18, 19, 20, 21, 23, 24, 25, 26}; |
-+ int32_t bounds1[] = {0, 5, 6, 10, 11, 17, 18, 20, 21}; |
- int32_t tag_lo[] = {UBRK_WORD_NONE, UBRK_WORD_LETTER, UBRK_WORD_NONE, UBRK_WORD_LETTER, |
- UBRK_WORD_NONE, UBRK_WORD_NUMBER, UBRK_WORD_NONE, |
-- UBRK_WORD_IDEO, UBRK_WORD_IDEO, UBRK_WORD_NONE, |
-- UBRK_WORD_KANA, UBRK_WORD_NONE, UBRK_WORD_KANA, UBRK_WORD_KANA}; |
-+ UBRK_WORD_IDEO, UBRK_WORD_NONE}; |
- |
- int32_t tag_hi[] = {UBRK_WORD_NONE_LIMIT, UBRK_WORD_LETTER_LIMIT, UBRK_WORD_NONE_LIMIT, UBRK_WORD_LETTER_LIMIT, |
- UBRK_WORD_NONE_LIMIT, UBRK_WORD_NUMBER_LIMIT, UBRK_WORD_NONE_LIMIT, |
-- UBRK_WORD_IDEO_LIMIT, UBRK_WORD_IDEO_LIMIT, UBRK_WORD_NONE_LIMIT, |
-- UBRK_WORD_KANA_LIMIT, UBRK_WORD_NONE_LIMIT, UBRK_WORD_KANA_LIMIT, UBRK_WORD_KANA_LIMIT}; |
-+ UBRK_WORD_IDEO_LIMIT, UBRK_WORD_NONE_LIMIT}; |
- |
- UErrorCode status=U_ZERO_ERROR; |
- |
-@@ -888,9 +893,11 @@ |
- |
- URegistryKey key = BreakIterator::registerInstance(ja_word, "xx", UBRK_WORD, status); |
- { |
-+#if 0 // With a dictionary based word breaking, ja_word is identical to root. |
- if (ja_word && *ja_word == *root_word) { |
- errln("japan not different from root"); |
- } |
-+#endif |
- } |
- |
- { |
---- source/test/intltest/rbbitst.cpp 2010-10-08 18:23:28.000000000 -0700 |
-+++ source/test/intltest/rbbitst.cpp 2011-01-21 14:12:45.180030000 -0800 |
-@@ -35,6 +35,8 @@ |
- #include <string.h> |
- #include <stdio.h> |
- #include <stdlib.h> |
-+#include "unicode/numfmt.h" |
-+#include "unicode/uscript.h" |
- |
- #define TEST_ASSERT(x) {if (!(x)) { \ |
- errln("Failure in file %s, line %d", __FILE__, __LINE__);}} |
-@@ -138,11 +140,13 @@ |
- if (exec) TestThaiBreaks(); break; |
- case 23: name = "TestTailoredBreaks"; |
- if (exec) TestTailoredBreaks(); break; |
-+ case 24: name = "TestTrieDictWithValue"; |
-+ if(exec) TestTrieDictWithValue(); break; |
- #else |
-- case 21: case 22: case 23: name = "skip"; |
-+ case 21: case 22: case 23: case 24: name = "skip"; |
- break; |
- #endif |
-- case 24: name = "TestDictRules"; |
-+ case 25: name = "TestDictRules"; |
- if (exec) TestDictRules(); break; |
- case 25: name = "TestBug5532"; |
- if (exec) TestBug5532(); break; |
-@@ -607,6 +611,8 @@ |
- |
- |
- void RBBITest::TestJapaneseWordBreak() { |
-+// TODO: Rewrite this test for a dictionary-based word breaking. |
-+#if 0 |
- UErrorCode status = U_ZERO_ERROR; |
- BITestData japaneseWordSelection(status); |
- |
-@@ -628,6 +634,7 @@ |
- |
- generalIteratorTest(*e, japaneseWordSelection); |
- delete e; |
-+#endif |
- } |
- |
- void RBBITest::TestTrieDict() { |
-@@ -849,6 +856,372 @@ |
- delete compact2; |
- } |
- |
-+/*TODO: delete later*/ |
-+inline void writeEnumerationToFile(StringEnumeration *enumer, char *filename){ |
-+ UErrorCode status = U_ZERO_ERROR; |
-+ FILE *outfile = fopen(filename,"w"); |
-+ UConverter *cvt = ucnv_open("UTF-8", &status); |
-+ if (U_FAILURE(status)) |
-+ return; |
-+ if(outfile != NULL){ |
-+ status = U_ZERO_ERROR; |
-+ const UnicodeString *word = enumer->snext(status); |
-+ while (word != NULL && U_SUCCESS(status)) { |
-+ char u8word[500]; |
-+ status = U_ZERO_ERROR; |
-+ ucnv_fromUChars(cvt, u8word, 500, word->getBuffer(), word->length(), |
-+ &status); |
-+ fprintf(outfile,"%s\n", u8word); |
-+ status = U_ZERO_ERROR; |
-+ word = enumer->snext(status); |
-+ } |
-+ fclose(outfile); |
-+ } |
-+ ucnv_close(cvt); |
-+} |
-+ |
-+// A very simple helper class to streamline the buffer handling in |
-+// TestTrieDictWithValue |
-+template<class T, size_t N> |
-+class AutoBuffer { |
-+ public: |
-+ AutoBuffer(size_t size) : buffer(stackBuffer) { |
-+ if (size > N) |
-+ buffer = new T[size]; |
-+ } |
-+ ~AutoBuffer() { |
-+ if (buffer != stackBuffer) |
-+ delete [] buffer; |
-+ } |
-+ T* elems() { |
-+ return buffer; |
-+ } |
-+ const T& operator[] (size_t i) const { |
-+ return buffer[i]; |
-+ } |
-+ T& operator[] (size_t i) { |
-+ return buffer[i]; |
-+ } |
-+ private: |
-+ T stackBuffer[N]; |
-+ T* buffer; |
-+ AutoBuffer(); |
-+}; |
-+ |
-+//---------------------------------------------------------------------------- |
-+// |
-+// TestTrieDictWithValue Test trie dictionaries with logprob values and |
-+// more than 2^16 nodes after compaction. |
-+// |
-+//---------------------------------------------------------------------------- |
-+void RBBITest::TestTrieDictWithValue() { |
-+ UErrorCode status = U_ZERO_ERROR; |
-+ |
-+ // |
-+ // Open and read the test data file. |
-+ // |
-+ const char *testDataDirectory = IntlTest::getSourceTestData(status); |
-+ const char *filename = "cjdict-truncated.txt"; |
-+ char testFileName[1000]; |
-+ if (testDataDirectory == NULL || strlen(testDataDirectory) + strlen(filename) + 10 >= sizeof(testFileName)) { |
-+ errln("Can't open test data. Path too long."); |
-+ return; |
-+ } |
-+ strcpy(testFileName, testDataDirectory); |
-+ strcat(testFileName, filename); |
-+ |
-+ // Items needing deleting at the end |
-+ MutableTrieDictionary *mutableDict = NULL; |
-+ CompactTrieDictionary *compactDict = NULL; |
-+ UnicodeSet *breaks = NULL; |
-+ UChar *testFile = NULL; |
-+ StringEnumeration *enumer1 = NULL; |
-+ StringEnumeration *enumer2 = NULL; |
-+ MutableTrieDictionary *mutable2 = NULL; |
-+ StringEnumeration *cloneEnum = NULL; |
-+ CompactTrieDictionary *compact2 = NULL; |
-+ NumberFormat *nf = NULL; |
-+ UText *originalText = NULL, *cloneText = NULL; |
-+ |
-+ const UnicodeString *originalWord = NULL; |
-+ const UnicodeString *cloneWord = NULL; |
-+ UChar *current; |
-+ UChar *word; |
-+ UChar uc; |
-+ int32_t wordLen; |
-+ int32_t wordCount; |
-+ int32_t testCount; |
-+ int32_t valueLen; |
-+ int counter = 0; |
-+ |
-+ int len; |
-+ testFile = ReadAndConvertFile(testFileName, len, NULL, status); |
-+ if (U_FAILURE(status)) { |
-+ goto cleanup; /* something went wrong, error already output */ |
-+ } |
-+ |
-+ mutableDict = new MutableTrieDictionary(0x0E1C, status, TRUE); |
-+ if (U_FAILURE(status)) { |
-+ errln("Error creating MutableTrieDictionary: %s\n", u_errorName(status)); |
-+ goto cleanup; |
-+ } |
-+ |
-+ breaks = new UnicodeSet; |
-+ breaks->add(0x000A); // Line Feed |
-+ breaks->add(0x000D); // Carriage Return |
-+ breaks->add(0x2028); // Line Separator |
-+ breaks->add(0x2029); // Paragraph Separator |
-+ breaks->add(0x0009); // Tab character |
-+ |
-+ // Now add each non-comment line of the file as a word. |
-+ current = testFile; |
-+ word = current; |
-+ uc = *current++; |
-+ wordLen = 0; |
-+ wordCount = 0; |
-+ nf = NumberFormat::createInstance(status); |
-+ |
-+ while (uc) { |
-+ UnicodeString ucharValue; |
-+ valueLen = 0; |
-+ |
-+ if (uc == 0x0023) { // #comment line, skip |
-+ while (uc && !breaks->contains(uc)) { |
-+ uc = *current++; |
-+ } |
-+ } |
-+ else{ |
-+ while (uc && !breaks->contains(uc)) { |
-+ ++wordLen; |
-+ uc = *current++; |
-+ } |
-+ if(uc == 0x0009){ //separator is a tab char, read in num after tab |
-+ uc = *current++; |
-+ while (uc && !breaks->contains(uc)) { |
-+ ucharValue.append(uc); |
-+ uc = *current++; |
-+ } |
-+ } |
-+ } |
-+ if (wordLen > 0) { |
-+ Formattable value((int32_t)0); |
-+ nf->parse(ucharValue.getTerminatedBuffer(), value, status); |
-+ |
-+ if(U_FAILURE(status)){ |
-+ errln("parsing of value failed when reading in dictionary\n"); |
-+ goto cleanup; |
-+ } |
-+ mutableDict->addWord(word, wordLen, status, value.getLong()); |
-+ if (U_FAILURE(status)) { |
-+ errln("Could not add word to mutable dictionary; status %s\n", u_errorName(status)); |
-+ goto cleanup; |
-+ } |
-+ wordCount += 1; |
-+ } |
-+ |
-+ // Find beginning of next line |
-+ while (uc && breaks->contains(uc)) { |
-+ uc = *current++; |
-+ } |
-+ word = current-1; |
-+ wordLen = 0; |
-+ } |
-+ |
-+ if (wordCount < 50) { |
-+ errln("Word count (%d) unreasonably small\n", wordCount); |
-+ goto cleanup; |
-+ } |
-+ |
-+ enumer1 = mutableDict->openWords(status); |
-+ if (U_FAILURE(status)) { |
-+ errln("Could not open mutable dictionary enumerator: %s\n", u_errorName(status)); |
-+ goto cleanup; |
-+ } |
-+ |
-+ testCount = 0; |
-+ if (wordCount != (testCount = enumer1->count(status))) { |
-+ errln("MutableTrieDictionary word count (%d) differs from file word count (%d), with status %s\n", |
-+ testCount, wordCount, u_errorName(status)); |
-+ goto cleanup; |
-+ } |
-+ |
-+ // Now compact it |
-+ compactDict = new CompactTrieDictionary(*mutableDict, status); |
-+ if (U_FAILURE(status)) { |
-+ errln("Failed to create CompactTrieDictionary: %s\n", u_errorName(status)); |
-+ goto cleanup; |
-+ } |
-+ |
-+ enumer2 = compactDict->openWords(status); |
-+ if (U_FAILURE(status)) { |
-+ errln("Could not open compact trie dictionary enumerator: %s\n", u_errorName(status)); |
-+ goto cleanup; |
-+ } |
-+ |
-+ |
-+ //delete later |
-+// writeEnumerationToFile(enumer1, "/home/jchye/mutable.txt"); |
-+// writeEnumerationToFile(enumer2, "/home/jchye/compact.txt"); |
-+ |
-+ enumer1->reset(status); |
-+ enumer2->reset(status); |
-+ |
-+ originalWord = enumer1->snext(status); |
-+ cloneWord = enumer2->snext(status); |
-+ while (U_SUCCESS(status) && originalWord != NULL && cloneWord != NULL) { |
-+ if (*originalWord != *cloneWord) { |
-+ errln("MutableTrieDictionary and CompactTrieDictionary word mismatch at %d, lengths are %d and %d\n", |
-+ counter, originalWord->length(), cloneWord->length()); |
-+ goto cleanup; |
-+ } |
-+ |
-+ // check if attached values of the same word in both dictionaries tally |
-+#if 0 |
-+ int32_t lengths1[originalWord->length()], lengths2[cloneWord->length()]; |
-+ uint16_t values1[originalWord->length()], values2[cloneWord->length()]; |
-+#endif |
-+ AutoBuffer<int32_t, 20> lengths1(originalWord->length()); |
-+ AutoBuffer<int32_t, 20> lengths2(cloneWord->length()); |
-+ AutoBuffer<uint16_t, 20> values1(originalWord->length()); |
-+ AutoBuffer<uint16_t, 20> values2(cloneWord->length()); |
-+ |
-+ originalText = utext_openConstUnicodeString(originalText, originalWord, &status); |
-+ cloneText = utext_openConstUnicodeString(cloneText, cloneWord, &status); |
-+ |
-+ int count1, count2; |
-+ mutableDict->matches(originalText, originalWord->length(), lengths1.elems(), count1, originalWord->length(), values1.elems()); |
-+ compactDict->matches(cloneText, cloneWord->length(), lengths2.elems(), count2, cloneWord->length(), values2.elems()); |
-+ |
-+ if(values1[count1-1] != values2[count2-1]){ |
-+ errln("Values of word %d in MutableTrieDictionary and CompactTrieDictionary do not match, with values %d and %d\n", |
-+ counter, values1[count1-1], values2[count2-1]); |
-+ goto cleanup; |
-+ } |
-+ |
-+ counter++; |
-+ originalWord = enumer1->snext(status); |
-+ cloneWord = enumer2->snext(status); |
-+ } |
-+ if (enumer1->getDynamicClassID() == enumer2->getDynamicClassID()) { |
-+ errln("CompactTrieEnumeration and MutableTrieEnumeration ClassIDs are the same"); |
-+ } |
-+ |
-+ delete enumer1; |
-+ enumer1 = NULL; |
-+ delete enumer2; |
-+ enumer2 = NULL; |
-+ |
-+ // Now un-compact it |
-+ mutable2 = compactDict->cloneMutable(status); |
-+ if (U_FAILURE(status)) { |
-+ errln("Could not clone CompactTrieDictionary to MutableTrieDictionary: %s\n", u_errorName(status)); |
-+ goto cleanup; |
-+ } |
-+ |
-+ cloneEnum = mutable2->openWords(status); |
-+ if (U_FAILURE(status)) { |
-+ errln("Could not create cloned mutable enumerator: %s\n", u_errorName(status)); |
-+ goto cleanup; |
-+ } |
-+ |
-+ if (wordCount != (testCount = cloneEnum->count(status))) { |
-+ errln("Cloned MutableTrieDictionary word count (%d) differs from file word count (%d), with status %s\n", |
-+ testCount, wordCount, u_errorName(status)); |
-+ goto cleanup; |
-+ } |
-+ |
-+ // Compact original dictionary to clone. Note that we can only compare the same kind of |
-+ // dictionary as the order of the enumerators is not guaranteed to be the same between |
-+ // different kinds |
-+ enumer1 = mutableDict->openWords(status); |
-+ if (U_FAILURE(status)) { |
-+ errln("Could not re-open mutable dictionary enumerator: %s\n", u_errorName(status)); |
-+ goto cleanup; |
-+ } |
-+ |
-+ counter = 0; |
-+ originalWord = enumer1->snext(status); |
-+ cloneWord = cloneEnum->snext(status); |
-+ while (U_SUCCESS(status) && originalWord != NULL && cloneWord != NULL) { |
-+ if (*originalWord != *cloneWord) { |
-+ errln("Original and cloned MutableTrieDictionary word mismatch\n"); |
-+ goto cleanup; |
-+ } |
-+ |
-+ // check if attached values of the same word in both dictionaries tally |
-+ AutoBuffer<int32_t, 20> lengths1(originalWord->length()); |
-+ AutoBuffer<int32_t, 20> lengths2(cloneWord->length()); |
-+ AutoBuffer<uint16_t, 20> values1(originalWord->length()); |
-+ AutoBuffer<uint16_t, 20> values2(cloneWord->length()); |
-+ originalText = utext_openConstUnicodeString(originalText, originalWord, &status); |
-+ cloneText = utext_openConstUnicodeString(cloneText, cloneWord, &status); |
-+ |
-+ int count1, count2; |
-+ mutableDict->matches(originalText, originalWord->length(), lengths1.elems(), count1, originalWord->length(), values1.elems()); |
-+ mutable2->matches(cloneText, cloneWord->length(), lengths2.elems(), count2, cloneWord->length(), values2.elems()); |
-+ |
-+ if(values1[count1-1] != values2[count2-1]){ |
-+ errln("Values of word %d in original and cloned MutableTrieDictionary do not match, with values %d and %d\n", |
-+ counter, values1[count1-1], values2[count2-1]); |
-+ goto cleanup; |
-+ } |
-+ |
-+ counter++; |
-+ |
-+ originalWord = enumer1->snext(status); |
-+ cloneWord = cloneEnum->snext(status); |
-+ } |
-+ |
-+ if (U_FAILURE(status)) { |
-+ errln("Enumeration failed: %s\n", u_errorName(status)); |
-+ goto cleanup; |
-+ } |
-+ |
-+ if (originalWord != cloneWord) { |
-+ errln("Original and cloned MutableTrieDictionary ended enumeration at different points\n"); |
-+ goto cleanup; |
-+ } |
-+ |
-+ // Test the data copying constructor for CompactTrieDict, and the data access APIs. |
-+ compact2 = new CompactTrieDictionary(compactDict->data(), status); |
-+ if (U_FAILURE(status)) { |
-+ errln("CompactTrieDictionary(const void *,...) failed\n"); |
-+ goto cleanup; |
-+ } |
-+ |
-+ if (compact2->dataSize() == 0) { |
-+ errln("CompactTrieDictionary->dataSize() == 0\n"); |
-+ goto cleanup; |
-+ } |
-+ |
-+ // Now count the words via the second dictionary |
-+ delete enumer1; |
-+ enumer1 = compact2->openWords(status); |
-+ if (U_FAILURE(status)) { |
-+ errln("Could not open compact trie dictionary 2 enumerator: %s\n", u_errorName(status)); |
-+ goto cleanup; |
-+ } |
-+ |
-+ if (wordCount != (testCount = enumer1->count(status))) { |
-+ errln("CompactTrieDictionary 2 word count (%d) differs from file word count (%d), with status %s\n", |
-+ testCount, wordCount, u_errorName(status)); |
-+ goto cleanup; |
-+ } |
-+ |
-+ cleanup: |
-+ delete compactDict; |
-+ delete mutableDict; |
-+ delete breaks; |
-+ delete[] testFile; |
-+ delete enumer1; |
-+ delete mutable2; |
-+ delete cloneEnum; |
-+ delete compact2; |
-+ utext_close(originalText); |
-+ utext_close(cloneText); |
-+ |
-+ |
-+} |
- |
- //---------------------------------------------------------------------------- |
- // |
-@@ -1870,8 +2243,15 @@ |
- // Don't break in runs of hiragana or runs of ideograph, where the latter includes \u3005 \u3007 \u303B (cldrbug #2009). |
- static const char jaWordText[] = "\\u79C1\\u9054\\u306B\\u4E00\\u3007\\u3007\\u3007\\u306E\\u30B3\\u30F3\\u30D4\\u30E5\\u30FC\\u30BF" |
- "\\u304C\\u3042\\u308B\\u3002\\u5948\\u3005\\u306F\\u30EF\\u30FC\\u30C9\\u3067\\u3042\\u308B\\u3002"; |
-+#if 0 |
- static const int32_t jaWordTOffsets[] = { 2, 3, 7, 8, 14, 17, 18, 20, 21, 24, 27, 28 }; |
- static const int32_t jaWordROffsets[] = { 1, 2, 3, 4, 5, 6, 7, 8, 14, 15, 16, 17, 18, 19, 20, 21, 24, 25, 26, 27, 28 }; |
-+#endif |
-+// There's no separate Japanese word break iterator. Root is the same as Japanese. |
-+// Our dictionary-based iterator has to be tweaked to better handle U+3005, |
-+// U+3007, U+300B and some other cases. |
-+static const int32_t jaWordTOffsets[] = { 1, 2, 3, 4, 5, 7, 8, 12, 13, 14, 15, 17, 18, 20, 21, 22, 23, 24, 25, 27, 28 }; |
-+static const int32_t jaWordROffsets[] = { 1, 2, 3, 4, 5, 7, 8, 12, 13, 14, 15, 17, 18, 20, 21, 22, 23, 24, 25, 27, 28 }; |
- |
- // UBreakIteratorType UBRK_SENTENCE, Locale "el" |
- // Add break after Greek question mark (cldrbug #2069). |
-@@ -2672,6 +3052,8 @@ |
- UnicodeSet *fNewlineSet; |
- UnicodeSet *fKatakanaSet; |
- UnicodeSet *fALetterSet; |
-+ // TODO(jungshik): Do we still need this change? |
-+ // UnicodeSet *fALetterSet; // matches ALetterPlus in word.txt |
- UnicodeSet *fMidNumLetSet; |
- UnicodeSet *fMidLetterSet; |
- UnicodeSet *fMidNumSet; |
-@@ -2680,6 +3062,7 @@ |
- UnicodeSet *fOtherSet; |
- UnicodeSet *fExtendSet; |
- UnicodeSet *fExtendNumLetSet; |
-+ UnicodeSet *fDictionaryCjkSet; |
- |
- RegexMatcher *fMatcher; |
- |
-@@ -2696,12 +3079,24 @@ |
- fCRSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = CR}]"), status); |
- fLFSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = LF}]"), status); |
- fNewlineSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = Newline}]"), status); |
-- fALetterSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = ALetter}]"), status); |
-+ fDictionaryCjkSet= new UnicodeSet("[[\\uac00-\\ud7a3][:Han:][:Hiragana:]]", status); |
-+ // Exclude Hangul syllables from ALetterSet during testing. |
-+ // Leave CJK dictionary characters out from the monkey tests! |
-+#if 0 |
-+ fALetterSet = new UnicodeSet("[\\p{Word_Break = ALetter}" |
-+ "[\\p{Line_Break = Complex_Context}" |
-+ "-\\p{Grapheme_Cluster_Break = Extend}" |
-+ "-\\p{Grapheme_Cluster_Break = Control}" |
-+ "]]", |
-+ status); |
-+#endif |
-+ fALetterSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = ALetter}]"), status); |
-+ fALetterSet->removeAll(*fDictionaryCjkSet); |
- fKatakanaSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = Katakana}]"), status); |
- fMidNumLetSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = MidNumLet}]"), status); |
- fMidLetterSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = MidLetter}]"), status); |
- fMidNumSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = MidNum}]"), status); |
-- fNumericSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = Numeric}]"), status); |
-+ fNumericSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = Numeric}[\\uff10-\\uff19]]"), status); |
- fFormatSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = Format}]"), status); |
- fExtendNumLetSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = ExtendNumLet}]"), status); |
- fExtendSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = Extend}]"), status); |
-@@ -2725,13 +3120,14 @@ |
- fOtherSet->removeAll(*fFormatSet); |
- fOtherSet->removeAll(*fExtendSet); |
- // Inhibit dictionary characters from being tested at all. |
-+ fOtherSet->removeAll(*fDictionaryCjkSet); |
- fOtherSet->removeAll(UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{LineBreak = Complex_Context}]"), status)); |
- |
- fSets->addElement(fCRSet, status); |
- fSets->addElement(fLFSet, status); |
- fSets->addElement(fNewlineSet, status); |
- fSets->addElement(fALetterSet, status); |
-- fSets->addElement(fKatakanaSet, status); |
-+ //fSets->addElement(fKatakanaSet, status); //TODO: work out how to test katakana |
- fSets->addElement(fMidLetterSet, status); |
- fSets->addElement(fMidNumLetSet, status); |
- fSets->addElement(fMidNumSet, status); |
-@@ -3978,6 +4374,7 @@ |
- for (i = bi->last(); i != BreakIterator::DONE; i = bi->previous()) { |
- count --; |
- if (forward[count] != i) { |
-+ printStringBreaks(ustr, expected, expectedcount); |
- test->errln("happy break test previous() failed: expected %d but got %d", |
- forward[count], i); |
- break; |
-@@ -4011,23 +4408,25 @@ |
- UErrorCode status = U_ZERO_ERROR; |
- // BreakIterator *bi = BreakIterator::createCharacterInstance(locale, status); |
- BreakIterator *bi = BreakIterator::createWordInstance(locale, status); |
-+ // Replaced any C+J characters in a row with a random sequence of characters |
-+ // of the same length to make our C+J segmentation not get in the way. |
- static const char *strlist[] = |
- { |
- "\\U000e0032\\u0097\\u0f94\\uc2d8\\u05f4\\U000e0031\\u060d", |
-- "\\U000e0037\\u4666\\u1202\\u003a\\U000e0031\\u064d\\u0bea\\u591c\\U000e0040\\u003b", |
-+ "\\U000e0037\\u2666\\u1202\\u003a\\U000e0031\\u064d\\u0bea\\u091c\\U000e0040\\u003b", |
- "\\u0589\\u3e99\\U0001d7f3\\U000e0074\\u1810\\u200e\\U000e004b\\u0027\\U000e0061\\u003a", |
- "\\u398c\\U000104a5\\U0001d173\\u102d\\u002e\\uca3b\\u002e\\u002c\\u5622", |
-- "\\u90ca\\u3588\\u009c\\u0953\\u194b", |
-+ "\\uac00\\u3588\\u009c\\u0953\\u194b", |
- "\\u200e\\U000e0072\\u0a4b\\U000e003f\\ufd2b\\u2027\\u002e\\u002e", |
- "\\u0602\\u2019\\ua191\\U000e0063\\u0a4c\\u003a\\ub4b5\\u003a\\u827f\\u002e", |
-- "\\u7f1f\\uc634\\u65f8\\u0944\\u04f2\\uacdf\\u1f9c\\u05f4\\u002e", |
-+ "\\u2f1f\\u1634\\u05f8\\u0944\\u04f2\\u0cdf\\u1f9c\\u05f4\\u002e", |
- "\\U000e0042\\u002e\\u0fb8\\u09ef\\u0ed1\\u2044", |
- "\\u003b\\u024a\\u102e\\U000e0071\\u0600", |
- "\\u2027\\U000e0067\\u0a47\\u00b7", |
- "\\u1fcd\\u002c\\u07aa\\u0027\\u11b0", |
- "\\u002c\\U000e003c\\U0001d7f4\\u003a\\u0c6f\\u0027", |
- "\\u0589\\U000e006e\\u0a42\\U000104a5", |
-- "\\u4f66\\ub523\\u003a\\uacae\\U000e0047\\u003a", |
-+ "\\u0f66\\u2523\\u003a\\u0cae\\U000e0047\\u003a", |
- "\\u003a\\u0f21\\u0668\\u0dab\\u003a\\u0655\\u00b7", |
- "\\u0027\\u11af\\U000e0057\\u0602", |
- "\\U0001d7f2\\U000e007\\u0004\\u0589", |
-@@ -4039,7 +4438,7 @@ |
- "\\u0be8\\u002e\\u0c68\\u066e\\u136d\\ufc99\\u59e7", |
- "\\u0233\\U000e0020\\u0a69\\u0d6a", |
- "\\u206f\\u0741\\ub3ab\\u2019\\ubcac\\u2019", |
-- "\\u58f4\\U000e0049\\u20e7\\u2027", |
-+ "\\u18f4\\U000e0049\\u20e7\\u2027", |
- "\\ub315\\U0001d7e5\\U000e0073\\u0c47\\u06f2\\u0c6a\\u0037\\u10fe", |
- "\\ua183\\u102d\\u0bec\\u003a", |
- "\\u17e8\\u06e7\\u002e\\u096d\\u003b", |
-@@ -4049,7 +4448,7 @@ |
- "\\U000e005d\\u2044\\u0731\\u0650\\u0061", |
- "\\u003a\\u0664\\u00b7\\u1fba", |
- "\\u003b\\u0027\\u00b7\\u47a3", |
-- "\\u2027\\U000e0067\\u0a42\\u00b7\\ubddf\\uc26c\\u003a\\u4186\\u041b", |
-+ "\\u2027\\U000e0067\\u0a42\\u00b7\\u4edf\\uc26c\\u003a\\u4186\\u041b", |
- "\\u0027\\u003a\\U0001d70f\\U0001d7df\\ubf4a\\U0001d7f5\\U0001d177\\u003a\\u0e51\\u1058\\U000e0058\\u00b7\\u0673", |
- "\\uc30d\\u002e\\U000e002c\\u0c48\\u003a\\ub5a1\\u0661\\u002c", |
- }; |
-@@ -4104,12 +4503,12 @@ |
- "\\U0001d7f2\\U000e007d\\u0004\\u0589", |
- "\\u82ab\\u17e8\\u0736\\u2019\\U0001d64d", |
- "\\u0e01\\ub55c\\u0a68\\U000e0037\\u0cd6\\u002c\\ub959", |
-- "\\U000e0065\\u302c\\uc986\\u09ee\\U000e0068", |
-+ "\\U000e0065\\u302c\\u09ee\\U000e0068", |
- "\\u0be8\\u002e\\u0c68\\u066e\\u136d\\ufc99\\u59e7", |
- "\\u0233\\U000e0020\\u0a69\\u0d6a", |
- "\\u206f\\u0741\\ub3ab\\u2019\\ubcac\\u2019", |
- "\\u58f4\\U000e0049\\u20e7\\u2027", |
-- "\\ub315\\U0001d7e5\\U000e0073\\u0c47\\u06f2\\u0c6a\\u0037\\u10fe", |
-+ "\\U0001d7e5\\U000e0073\\u0c47\\u06f2\\u0c6a\\u0037\\u10fe", |
- "\\ua183\\u102d\\u0bec\\u003a", |
- "\\u17e8\\u06e7\\u002e\\u096d\\u003b", |
- "\\u003a\\u0e57\\u0fad\\u002e", |
---- source/test/intltest/rbbitst.h 2010-07-22 17:15:37.000000000 -0700 |
-+++ source/test/intltest/rbbitst.h 2011-01-21 14:12:45.152007000 -0800 |
-@@ -70,6 +70,7 @@ |
- void TestBug5775(); |
- void TestThaiBreaks(); |
- void TestTailoredBreaks(); |
-+ void TestTrieDictWithValue(); |
- void TestDictRules(); |
- void TestBug5532(); |
- |
---- source/test/testdata/rbbitst.txt 2010-07-28 17:18:28.000000000 -0700 |
-+++ source/test/testdata/rbbitst.txt 2011-01-21 14:12:45.221011000 -0800 |
-@@ -161,7 +161,23 @@ |
- <data>•abc<200>\U0001D800•def<200>\U0001D3FF• •</data> |
- |
- # Hiragana & Katakana stay together, but separates from each other and Latin. |
--<data>•abc<200>\N{HIRAGANA LETTER SMALL A}<300>\N{HIRAGANA LETTER VU}\N{COMBINING ACUTE ACCENT}<300>\N{HIRAGANA ITERATION MARK}<300>\N{KATAKANA LETTER SMALL A}\N{KATAKANA ITERATION MARK}\N{HALFWIDTH KATAKANA LETTER WO}\N{HALFWIDTH KATAKANA LETTER N}<300>def<200>#•</data> |
-+# *** what to do about theoretical combos of chars? i.e. hiragana + accent |
-+#<data>•abc<200>\N{HIRAGANA LETTER SMALL A}<300>\N{HIRAGANA LETTER VU}\N{COMBINING ACUTE ACCENT}<300>\N{HIRAGANA ITERATION MARK}<300>\N{KATAKANA LETTER SMALL A}\N{KATAKANA ITERATION MARK}\N{HALFWIDTH KATAKANA LETTER WO}\N{HALFWIDTH KATAKANA LETTER N}<300>def<200>#•</data> |
-+ |
-+# test normalization/dictionary handling of halfwidth katakana: same dictionary phrase in fullwidth and halfwidth |
-+<data>•芽キャベツ<400>芽キャベツ<400></data> |
-+ |
-+# more Japanese tests |
-+# TODO: Currently, U+30FC and other characters (script=common) in the Hiragana |
-+# and the Katakana block are not treated correctly. Enable this later. |
-+#<data>•どー<400>せ<400>日本語<400>を<400>勉強<400>する<400>理由<400>について<400> •て<400>こと<400>は<400>我<400>でも<400>知<400>ら<400>も<400>い<400>こと<400>なん<400>だ<400>。•</data> |
-+<data>•日本語<400>を<400>勉強<400>する<400>理由<400>について<400> •て<400>こと<400>は<400>我<400>でも<400>知<400>ら<400>も<400>い<400>こと<400>なん<400>だ<400>。•</data> |
-+ |
-+# Testing of word boundary for dictionary word containing both kanji and kana |
-+<data>•中だるみ<400>蔵王の森<400>ウ離島<400></data> |
-+ |
-+# Testing of Chinese segmentation (taken from a Chinese news article) |
-+<data>•400<100>余<400>名<400>中央<400>委员<400>和<400>中央<400>候补<400>委员<400>都<400>领<400>到了<400>“•推荐<400>票<400>”•,•有<400>资格<400>在<400>200<100>多<400>名<400>符合<400>条件<400>的<400>63<100>岁<400>以下<400>中共<400>正<400>部<400>级<400>干部<400>中<400>,•选出<400>他们<400>属意<400>的<400>中央<400>政治局<400>委员<400>以<400>向<400>政治局<400>常委<400>会<400>举荐<400>。•</data> |
- |
- # Words with interior formatting characters |
- <data>•def\N{COMBINING ACUTE ACCENT}\N{SYRIAC ABBREVIATION MARK}ghi<200> •</data> |
-@@ -169,6 +185,8 @@ |
- # to test for bug #4097779 |
- <data>•aa\N{COMBINING GRAVE ACCENT}a<200> •</data> |
- |
-+# fullwidth numeric, midletter characters etc should be treated like their halfwidth counterparts |
-+<data>•ISN'T<200> •19<100>日<400></data> |
- |
- # to test for bug #4098467 |
- # What follows is a string of Korean characters (I found it in the Yellow Pages |
-@@ -178,9 +196,15 @@ |
- # precomposed syllables... |
- <data>•\uc0c1\ud56d<200> •\ud55c\uc778<200> •\uc5f0\ud569<200> •\uc7a5\ub85c\uad50\ud68c<200> •\u1109\u1161\u11bc\u1112\u1161\u11bc<200> •\u1112\u1161\u11ab\u110b\u1175\u11ab<200> •\u110b\u1167\u11ab\u1112\u1161\u11b8<200> •\u110c\u1161\u11bc\u1105\u1169\u1100\u116d\u1112\u116c<200> •</data> |
- |
--<data>•abc<200>\u4e01<400>\u4e02<400>\u3005<200>\u4e03<400>\u4e03<400>abc<200> •</data> |
-+# more Korean tests (Jamo not tested here, not counted as dictionary characters) |
-+# Disable them now because we don't include a Korean dictionary. |
-+#<data>•\ud55c\uad6d<200>\ub300\ud559\uad50<200>\uc790\uc5f0<200>\uacfc\ud559<200>\ub300\ud559<200>\ubb3c\ub9ac\ud559\uacfc<200></data> |
-+#<data>•\ud604\uc7ac<200>\ub294<200> •\uac80\ucc30<200>\uc774<200> •\ubd84\uc2dd<200>\ud68c\uacc4<200>\ubb38\uc81c<200>\ub97c<200> •\uc870\uc0ac<200>\ud560<200> •\uac00\ub2a5\uc131<200>\uc740<200> •\uc5c6\ub2e4<200>\u002e•</data> |
-+ |
-+<data>•abc<200>\u4e01<400>\u4e02<400>\u3005<400>\u4e03\u4e03<400>abc<200> •</data> |
-+ |
-+<data>•\u06c9<200>\uc799<200>\ufffa•</data> |
- |
--<data>•\u06c9\uc799\ufffa<200></data> |
- |
- # |
- # Try some words from other scripts. |
-@@ -491,8 +515,7 @@ |
- <data>•\uc0c1•\ud56d •\ud55c•\uc778 •\uc5f0•\ud569 •\uc7a5•\ub85c•\uad50•\ud68c•</data> |
- |
- # conjoining jamo... |
--# TODO: rules update needed |
--#<data>•\u1109\u1161\u11bc•\u1112\u1161\u11bc •\u1112\u1161\u11ab•\u110b\u1175\u11ab #•\u110b\u1167\u11ab•\u1112\u1161\u11b8 •\u110c\u1161\u11bc•\u1105\u1169•\u1100\u116d•\u1112\u116c•</data> |
-+<data>•\u1109\u1161\u11bc•\u1112\u1161\u11bc •\u1112\u1161\u11ab•\u110b\u1175\u11ab •\u110b\u1167\u11ab•\u1112\u1161\u11b8 •\u110c\u1161\u11bc•\u1105\u1169•\u1100\u116d•\u1112\u116c•</data> |
- |
- # to test for bug #4117554: Fullwidth .!? should be treated as postJwrd |
- <data>•\u4e01\uff0e•\u4e02\uff01•\u4e03\uff1f•</data> |
---- source/test/testdata/testaliases.txt 2009-11-12 13:53:42.000000000 -0800 |
-+++ source/test/testdata/testaliases.txt 2011-01-21 14:12:45.204005000 -0800 |
-@@ -28,7 +28,7 @@ |
- LocaleScript:alias { "/ICUDATA/ja/LocaleScript" } |
- |
- // aliasing using position |
-- boundaries:alias { "/ICUDATA-brkitr/ja" } // Referencing corresponding resource in another bundle |
-+ boundaries:alias { "/ICUDATA-brkitr/th" } // Referencing corresponding resource in another bundle |
- |
- // aliasing arrays |
- zoneTests { |
---- source/tools/genctd/genctd.cpp 2009-08-04 14:09:17.000000000 -0700 |
-+++ source/tools/genctd/genctd.cpp 2011-01-21 14:12:45.564923000 -0800 |
-@@ -1,6 +1,6 @@ |
- /* |
- ********************************************************************** |
--* Copyright (C) 2002-2009, International Business Machines |
-+* Copyright (C) 2002-2010, International Business Machines |
- * Corporation and others. All Rights Reserved. |
- ********************************************************************** |
- * |
-@@ -34,12 +34,15 @@ |
- #include "unicode/udata.h" |
- #include "unicode/putil.h" |
- |
-+//#include "unicode/ustdio.h" |
-+ |
- #include "uoptions.h" |
- #include "unewdata.h" |
- #include "ucmndata.h" |
- #include "rbbidata.h" |
- #include "triedict.h" |
- #include "cmemory.h" |
-+#include "uassert.h" |
- |
- #include <stdio.h> |
- #include <stdlib.h> |
-@@ -199,147 +202,191 @@ |
- long wordFileSize; |
- FILE *file; |
- char *wordBufferC; |
-- |
-+ MutableTrieDictionary *mtd = NULL; |
-+ |
- file = fopen(wordFileName, "rb"); |
-- if( file == 0 ) { |
-- fprintf(stderr, "Could not open file \"%s\"\n", wordFileName); |
-- exit(-1); |
-- } |
-- fseek(file, 0, SEEK_END); |
-- wordFileSize = ftell(file); |
-- fseek(file, 0, SEEK_SET); |
-- wordBufferC = new char[wordFileSize+10]; |
-- |
-- result = (long)fread(wordBufferC, 1, wordFileSize, file); |
-- if (result != wordFileSize) { |
-- fprintf(stderr, "Error reading file \"%s\"\n", wordFileName); |
-- exit (-1); |
-- } |
-- wordBufferC[wordFileSize]=0; |
-- fclose(file); |
-- |
-- // |
-- // Look for a Unicode Signature (BOM) on the word file |
-- // |
-- int32_t signatureLength; |
-- const char * wordSourceC = wordBufferC; |
-- const char* encoding = ucnv_detectUnicodeSignature( |
-- wordSourceC, wordFileSize, &signatureLength, &status); |
-- if (U_FAILURE(status)) { |
-- exit(status); |
-- } |
-- if(encoding!=NULL ){ |
-- wordSourceC += signatureLength; |
-- wordFileSize -= signatureLength; |
-- } |
-- |
-- // |
-- // Open a converter to take the rule file to UTF-16 |
-- // |
-- UConverter* conv; |
-- conv = ucnv_open(encoding, &status); |
-- if (U_FAILURE(status)) { |
-- fprintf(stderr, "ucnv_open: ICU Error \"%s\"\n", u_errorName(status)); |
-- exit(status); |
-- } |
-- |
-- // |
-- // Convert the words to UChar. |
-- // Preflight first to determine required buffer size. |
-- // |
-- uint32_t destCap = ucnv_toUChars(conv, |
-- NULL, // dest, |
-- 0, // destCapacity, |
-- wordSourceC, |
-- wordFileSize, |
-- &status); |
-- if (status != U_BUFFER_OVERFLOW_ERROR) { |
-- fprintf(stderr, "ucnv_toUChars: ICU Error \"%s\"\n", u_errorName(status)); |
-- exit(status); |
-- }; |
-- |
-- status = U_ZERO_ERROR; |
-- UChar *wordSourceU = new UChar[destCap+1]; |
-- ucnv_toUChars(conv, |
-- wordSourceU, // dest, |
-- destCap+1, |
-- wordSourceC, |
-- wordFileSize, |
-- &status); |
-- if (U_FAILURE(status)) { |
-- fprintf(stderr, "ucnv_toUChars: ICU Error \"%s\"\n", u_errorName(status)); |
-- exit(status); |
-- }; |
-- ucnv_close(conv); |
-- |
-- // Get rid of the original file buffer |
-- delete[] wordBufferC; |
-- |
-- // Create a MutableTrieDictionary, and loop through all the lines, inserting |
-- // words. |
-- |
-- // First, pick a median character. |
-- UChar *current = wordSourceU + (destCap/2); |
-- UChar uc = *current++; |
-- UnicodeSet breaks; |
-- breaks.add(0x000A); // Line Feed |
-- breaks.add(0x000D); // Carriage Return |
-- breaks.add(0x2028); // Line Separator |
-- breaks.add(0x2029); // Paragraph Separator |
-- |
-- do { |
-- // Look for line break |
-- while (uc && !breaks.contains(uc)) { |
-- uc = *current++; |
-- } |
-- // Now skip to first non-line-break |
-- while (uc && breaks.contains(uc)) { |
-- uc = *current++; |
-+ if( file == 0 ) { //cannot find file |
-+ //create 1-line dummy file: ie 1 char, 1 value |
-+ UNewDataMemory *pData; |
-+ char msg[1024]; |
-+ |
-+ /* write message with just the name */ |
-+ sprintf(msg, "%s not found, genctd writes dummy %s", wordFileName, outFileName); |
-+ fprintf(stderr, "%s\n", msg); |
-+ |
-+ UChar c = 0x0020; |
-+ mtd = new MutableTrieDictionary(c, status, TRUE); |
-+ mtd->addWord(&c, 1, status, 1); |
-+ |
-+ } else { //read words in from input file |
-+ fseek(file, 0, SEEK_END); |
-+ wordFileSize = ftell(file); |
-+ fseek(file, 0, SEEK_SET); |
-+ wordBufferC = new char[wordFileSize+10]; |
-+ |
-+ result = (long)fread(wordBufferC, 1, wordFileSize, file); |
-+ if (result != wordFileSize) { |
-+ fprintf(stderr, "Error reading file \"%s\"\n", wordFileName); |
-+ exit (-1); |
- } |
-- } |
-- while (uc && (breaks.contains(uc) || u_isspace(uc))); |
-- |
-- MutableTrieDictionary *mtd = new MutableTrieDictionary(uc, status); |
-+ wordBufferC[wordFileSize]=0; |
-+ fclose(file); |
- |
-- if (U_FAILURE(status)) { |
-- fprintf(stderr, "new MutableTrieDictionary: ICU Error \"%s\"\n", u_errorName(status)); |
-- exit(status); |
-- } |
-+ // |
-+ // Look for a Unicode Signature (BOM) on the word file |
-+ // |
-+ int32_t signatureLength; |
-+ const char * wordSourceC = wordBufferC; |
-+ const char* encoding = ucnv_detectUnicodeSignature( |
-+ wordSourceC, wordFileSize, &signatureLength, &status); |
-+ if (U_FAILURE(status)) { |
-+ exit(status); |
-+ } |
-+ if(encoding!=NULL ){ |
-+ wordSourceC += signatureLength; |
-+ wordFileSize -= signatureLength; |
-+ } |
- |
-- // Now add the words. Words are non-space characters at the beginning of |
-- // lines, and must be at least one UChar. |
-- current = wordSourceU; |
-- UChar *candidate = current; |
-- uc = *current++; |
-- int32_t length = 0; |
-- |
-- while (uc) { |
-- while (uc && !u_isspace(uc)) { |
-- ++length; |
-- uc = *current++; |
-+ // |
-+ // Open a converter to take the rule file to UTF-16 |
-+ // |
-+ UConverter* conv; |
-+ conv = ucnv_open(encoding, &status); |
-+ if (U_FAILURE(status)) { |
-+ fprintf(stderr, "ucnv_open: ICU Error \"%s\"\n", u_errorName(status)); |
-+ exit(status); |
- } |
-- if (length > 0) { |
-- mtd->addWord(candidate, length, status); |
-- if (U_FAILURE(status)) { |
-- fprintf(stderr, "MutableTrieDictionary::addWord: ICU Error \"%s\"\n", |
-- u_errorName(status)); |
-- exit(status); |
-+ |
-+ // |
-+ // Convert the words to UChar. |
-+ // Preflight first to determine required buffer size. |
-+ // |
-+ uint32_t destCap = ucnv_toUChars(conv, |
-+ NULL, // dest, |
-+ 0, // destCapacity, |
-+ wordSourceC, |
-+ wordFileSize, |
-+ &status); |
-+ if (status != U_BUFFER_OVERFLOW_ERROR) { |
-+ fprintf(stderr, "ucnv_toUChars: ICU Error \"%s\"\n", u_errorName(status)); |
-+ exit(status); |
-+ }; |
-+ |
-+ status = U_ZERO_ERROR; |
-+ UChar *wordSourceU = new UChar[destCap+1]; |
-+ ucnv_toUChars(conv, |
-+ wordSourceU, // dest, |
-+ destCap+1, |
-+ wordSourceC, |
-+ wordFileSize, |
-+ &status); |
-+ if (U_FAILURE(status)) { |
-+ fprintf(stderr, "ucnv_toUChars: ICU Error \"%s\"\n", u_errorName(status)); |
-+ exit(status); |
-+ }; |
-+ ucnv_close(conv); |
-+ |
-+ // Get rid of the original file buffer |
-+ delete[] wordBufferC; |
-+ |
-+ // Create a MutableTrieDictionary, and loop through all the lines, inserting |
-+ // words. |
-+ |
-+ // First, pick a median character. |
-+ UChar *current = wordSourceU + (destCap/2); |
-+ UChar uc = *current++; |
-+ UnicodeSet breaks; |
-+ breaks.add(0x000A); // Line Feed |
-+ breaks.add(0x000D); // Carriage Return |
-+ breaks.add(0x2028); // Line Separator |
-+ breaks.add(0x2029); // Paragraph Separator |
-+ |
-+ do { |
-+ // Look for line break |
-+ while (uc && !breaks.contains(uc)) { |
-+ uc = *current++; |
-+ } |
-+ // Now skip to first non-line-break |
-+ while (uc && breaks.contains(uc)) { |
-+ uc = *current++; |
- } |
- } |
-- // Find beginning of next line |
-- while (uc && !breaks.contains(uc)) { |
-- uc = *current++; |
-+ while (uc && (breaks.contains(uc) || u_isspace(uc))); |
-+ |
-+ mtd = new MutableTrieDictionary(uc, status); |
-+ |
-+ if (U_FAILURE(status)) { |
-+ fprintf(stderr, "new MutableTrieDictionary: ICU Error \"%s\"\n", u_errorName(status)); |
-+ exit(status); |
- } |
-- while (uc && breaks.contains(uc)) { |
-- uc = *current++; |
-+ |
-+ // Now add the words. Words are non-space characters at the beginning of |
-+ // lines, and must be at least one UChar. If a word has an associated value, |
-+ // the value should follow the word on the same line after a tab character. |
-+ current = wordSourceU; |
-+ UChar *candidate = current; |
-+ uc = *current++; |
-+ int32_t length = 0; |
-+ int count = 0; |
-+ |
-+ while (uc) { |
-+ while (uc && !u_isspace(uc)) { |
-+ ++length; |
-+ uc = *current++; |
-+ } |
-+ |
-+ UnicodeString valueString; |
-+ UChar candidateValue; |
-+ if(uc == 0x0009){ //separator is a tab char, read in number after space |
-+ while (uc && u_isspace(uc)) { |
-+ uc = *current++; |
-+ } |
-+ while (uc && !u_isspace(uc)) { |
-+ valueString.append(uc); |
-+ uc = *current++; |
-+ } |
-+ } |
-+ |
-+ if (length > 0) { |
-+ count++; |
-+ if(valueString.length() > 0){ |
-+ mtd->setValued(TRUE); |
-+ |
-+ uint32_t value = 0; |
-+ char* s = new char[valueString.length()]; |
-+ valueString.extract(0,valueString.length(), s, valueString.length()); |
-+ int n = sscanf(s, "%ud", &value); |
-+ U_ASSERT(n == 1); |
-+ U_ASSERT(value >= 0); |
-+ mtd->addWord(candidate, length, status, (uint16_t)value); |
-+ delete[] s; |
-+ } else { |
-+ mtd->addWord(candidate, length, status); |
-+ } |
-+ |
-+ if (U_FAILURE(status)) { |
-+ fprintf(stderr, "MutableTrieDictionary::addWord: ICU Error \"%s\" at line %d in input file\n", |
-+ u_errorName(status), count); |
-+ exit(status); |
-+ } |
-+ } |
-+ |
-+ // Find beginning of next line |
-+ while (uc && !breaks.contains(uc)) { |
-+ uc = *current++; |
-+ } |
-+ // Find next non-line-breaking character |
-+ while (uc && breaks.contains(uc)) { |
-+ uc = *current++; |
-+ } |
-+ candidate = current-1; |
-+ length = 0; |
- } |
-- candidate = current-1; |
-- length = 0; |
-+ |
-+ // Get rid of the Unicode text buffer |
-+ delete[] wordSourceU; |
- } |
- |
-- // Get rid of the Unicode text buffer |
-- delete[] wordSourceU; |
-- |
- // Now, create a CompactTrieDictionary from the mutable dictionary |
- CompactTrieDictionary *ctd = new CompactTrieDictionary(*mtd, status); |
- if (U_FAILURE(status)) { |
-@@ -393,4 +440,3 @@ |
- |
- #endif /* #if !UCONFIG_NO_BREAK_ITERATION */ |
- } |
-- |
---- source/tools/genctd/Makefile.in 2006-12-16 13:07:01.000000000 -0800 |
-+++ source/tools/genctd/Makefile.in 2011-01-21 14:12:45.555920000 -0800 |
-@@ -23,13 +23,13 @@ |
- ## Extra files to remove for 'make clean' |
- CLEANFILES = *~ $(DEPS) $(MAN_FILES) |
- |
--## Target information |
-+## Target informationcd |
- TARGET = $(BINDIR)/$(TARGET_STUB_NAME)$(EXEEXT) |
- |
- ifneq ($(top_builddir),$(top_srcdir)) |
- CPPFLAGS += -I$(top_builddir)/common |
- endif |
--CPPFLAGS += -I$(top_srcdir)/common -I$(srcdir)/../toolutil |
-+CPPFLAGS += -I$(top_srcdir)/common -I$(srcdir)/../toolutil -I$(top_srcdir)/i18n |
- LIBS = $(LIBICUTOOLUTIL) $(LIBICUI18N) $(LIBICUUC) $(DEFAULT_LIBS) $(LIB_M) |
- |
- OBJECTS = genctd.o |