CJK segmentation patch for ICU 4.6...

icu46/source/test/intltest/rbbitst.cpp

 /********************************************************************
  * COPYRIGHT:
  * Copyright (c) 1999-2010, International Business Machines Corporation and
  * others. All Rights Reserved.
  ********************************************************************/
 /************************************************************************
 *   Date        Name        Description
 *   12/15/99    Madhu        Creation.
 *   01/12/2000  Madhu        Updated for changed API and added new tests
 ************************************************************************/
@@ skipping 17 matching lines @@
 #include "unicode/utext.h"
 #include "intltest.h"
 #include "rbbitst.h"
 #include <string.h>
 #include "uvector.h"
 #include "uvectr32.h"
 #include "triedict.h"
 #include <string.h>
 #include <stdio.h>
 #include <stdlib.h>
+#include "unicode/numfmt.h"
+#include "unicode/uscript.h"
 
 #define TEST_ASSERT(x) {if (!(x)) { \
     errln("Failure in file %s, line %d", __FILE__, __LINE__);}}
 
 #define TEST_ASSERT_SUCCESS(errcode) { if (U_FAILURE(errcode)) { \
     errcheckln(errcode, "Failure in file %s, line %d, status = \"%s\"", __FILE__, __LINE__, u_errorName(errcode));}}
 
 
 //---------------------------------------------
 // runIndexedTest
@@ skipping 83 matching lines @@
         case 20: name = "TestTrieDict";
             if(exec) TestTrieDict();                           break;
 
 #if !UCONFIG_NO_FILE_IO
         case 21: name = "TestBug5775";
             if (exec) TestBug5775();                           break;
         case 22: name = "TestThaiBreaks";
             if (exec) TestThaiBreaks();                        break;
         case 23: name = "TestTailoredBreaks";
             if (exec) TestTailoredBreaks();                    break;
+        case 24: name = "TestTrieDictWithValue";
+            if(exec) TestTrieDictWithValue();                  break;
 #else
-        case 21: case 22: case 23: name = "skip";
+        case 21: case 22: case 23: case 24: name = "skip";
             break;
 #endif
-        case 24: name = "TestDictRules";
+        case 25: name = "TestDictRules";
             if (exec) TestDictRules();                         break;
         case 25: name = "TestBug5532";
             if (exec) TestBug5532();                           break;
         default: name = ""; break; //needed to end loop
     }
 }
 
 
 //---------------------------------------------------------------------------
 //
@@ skipping 444 matching lines @@
     startOfSecondWord = bi->following(0);
     if (startOfSecondWord != 4) {
         errln("Fail at file %s, line %d expected start of word at 4, got %d",
             __FILE__, __LINE__, startOfSecondWord);
     }
     delete bi;
 }
 
 
 void RBBITest::TestJapaneseWordBreak() {
+// TODO: Rewrite this test for a dictionary-based word breaking.
+#if 0
     UErrorCode status = U_ZERO_ERROR;
     BITestData   japaneseWordSelection(status);
 
     ADD_DATACHUNK(japaneseWordSelection, NULL, 0, status);           // Break at start of data
     ADD_DATACHUNK(japaneseWordSelection, "\\u4ECA\\u65E5", 400, status); //2
     ADD_DATACHUNK(japaneseWordSelection, "\\u306F\\u3044\\u3044", 300, status); //5
     ADD_DATACHUNK(japaneseWordSelection, "\\u5929\\u6C17", 400, status); //7
     ADD_DATACHUNK(japaneseWordSelection, "\\u3067\\u3059\\u306D", 300, status); //10
     ADD_DATACHUNK(japaneseWordSelection, "\\u3002", 0, status); //11
     ADD_DATACHUNK(japaneseWordSelection, "\\u000D\\u000A", 0, status); //12
 
     RuleBasedBreakIterator* e = (RuleBasedBreakIterator *)BreakIterator::createWordInstance(
         Locale("ja"), status);
     if (U_FAILURE(status))
     {
         errcheckln(status, "Failed to create the BreakIterator for Japanese locale in TestJapaneseWordBreak.\n");
         return;
     }
 
     generalIteratorTest(*e, japaneseWordSelection);
     delete e;
+#endif
 }
 
 void RBBITest::TestTrieDict() {
     UErrorCode      status  = U_ZERO_ERROR;
 
     //
     //  Open and read the test data file.
     //
     const char *testDataDirectory = IntlTest::getSourceTestData(status);
     char testFileName[1000];
@@ skipping 201 matching lines @@
     delete compactDict;
     delete mutableDict;
     delete breaks;
     delete[] testFile;
     delete enumer1;
     delete mutable2;
     delete cloneEnum;
     delete compact2;
 }
 
+/*TODO: delete later*/
+inline void writeEnumerationToFile(StringEnumeration *enumer, char *filename){
+    UErrorCode      status  = U_ZERO_ERROR;
+    FILE *outfile = fopen(filename,"w");
+    UConverter *cvt = ucnv_open("UTF-8", &status);
+    if (U_FAILURE(status))
+        return;
+    if(outfile != NULL){
+        status = U_ZERO_ERROR;
+        const UnicodeString *word = enumer->snext(status);
+        while (word != NULL && U_SUCCESS(status)) {
+            char u8word[500];
+            status = U_ZERO_ERROR;
+            ucnv_fromUChars(cvt, u8word, 500, word->getBuffer(), word->length(),
+                    &status);
+            fprintf(outfile,"%s\n", u8word);
+            status = U_ZERO_ERROR;
+            word = enumer->snext(status);
+        }
+        fclose(outfile);
+    }
+    ucnv_close(cvt);
+}
+
+// A very simple helper class to streamline the buffer handling in
+// TestTrieDictWithValue
+template<class T, size_t N>
+class AutoBuffer {
+ public:
+  AutoBuffer(size_t size) : buffer(stackBuffer) {
+    if (size > N)
+      buffer = new T[size];
+  }
+  ~AutoBuffer() {
+    if (buffer != stackBuffer) 
+      delete [] buffer;
+  }
+  T* elems() {
+    return buffer;
+  }
+  const T& operator[] (size_t i) const {
+    return buffer[i];
+  }
+  T& operator[] (size_t i) {
+    return buffer[i];
+  }
+ private:
+  T stackBuffer[N]; 
+  T* buffer;
+  AutoBuffer();
+};
+
+//----------------------------------------------------------------------------
+//
+// TestTrieDictWithValue    Test trie dictionaries with logprob values and 
+// more than 2^16 nodes after compaction.
+//
+//----------------------------------------------------------------------------
+void RBBITest::TestTrieDictWithValue() {
+    UErrorCode      status  = U_ZERO_ERROR;
+
+    //
+    //  Open and read the test data file.
+    //
+    const char *testDataDirectory = IntlTest::getSourceTestData(status);
+    const char *filename = "cjdict-truncated.txt";
+    char testFileName[1000];
+    if (testDataDirectory == NULL || strlen(testDataDirectory) + strlen(filename) + 10 >= sizeof(testFileName)) {
+        errln("Can't open test data.  Path too long.");
+        return;
+    }
+    strcpy(testFileName, testDataDirectory);
+    strcat(testFileName, filename);
+
+    // Items needing deleting at the end
+    MutableTrieDictionary *mutableDict = NULL;
+    CompactTrieDictionary *compactDict = NULL;
+    UnicodeSet            *breaks      = NULL;
+    UChar                 *testFile    = NULL;
+    StringEnumeration     *enumer1     = NULL;
+    StringEnumeration     *enumer2     = NULL;
+    MutableTrieDictionary *mutable2    = NULL;
+    StringEnumeration     *cloneEnum   = NULL;
+    CompactTrieDictionary *compact2    = NULL;
+    NumberFormat          *nf           = NULL;
+    UText *originalText = NULL, *cloneText = NULL;
+
+    const UnicodeString *originalWord = NULL;
+    const UnicodeString *cloneWord    = NULL;
+    UChar *current;
+    UChar *word;
+    UChar uc;
+    int32_t wordLen;
+    int32_t wordCount;
+    int32_t testCount;
+    int32_t valueLen;
+    int counter = 0;
+
+    int    len;
+    testFile = ReadAndConvertFile(testFileName, len, NULL, status);
+    if (U_FAILURE(status)) {
+        goto cleanup; /* something went wrong, error already output */
+    }
+
+    mutableDict = new MutableTrieDictionary(0x0E1C, status, TRUE);
+    if (U_FAILURE(status)) {
+        errln("Error creating MutableTrieDictionary: %s\n", u_errorName(status));
+        goto cleanup;
+    }
+
+    breaks = new UnicodeSet;
+    breaks->add(0x000A);     // Line Feed
+    breaks->add(0x000D);     // Carriage Return
+    breaks->add(0x2028);     // Line Separator
+    breaks->add(0x2029);     // Paragraph Separator
+    breaks->add(0x0009);     // Tab character
+
+    // Now add each non-comment line of the file as a word.
+    current = testFile;
+    word = current;
+    uc = *current++;
+    wordLen = 0;
+    wordCount = 0;
+    nf = NumberFormat::createInstance(status);
+
+    while (uc) {
+        UnicodeString ucharValue;
+        valueLen = 0;
+
+        if (uc == 0x0023) {     // #comment line, skip
+            while (uc && !breaks->contains(uc)) {
+                uc = *current++;
+            }
+        }
+        else{
+            while (uc && !breaks->contains(uc)) {
+                ++wordLen;
+                uc = *current++;
+            }
+            if(uc == 0x0009){ //separator is a tab char, read in num after tab
+                uc = *current++;
+                while (uc && !breaks->contains(uc)) {
+                    ucharValue.append(uc);
+                    uc = *current++;
+                }
+            }
+        }
+        if (wordLen > 0) {
+            Formattable value((int32_t)0);
+            nf->parse(ucharValue.getTerminatedBuffer(), value, status);
+            
+            if(U_FAILURE(status)){
+                errln("parsing of value failed when reading in dictionary\n");
+                goto cleanup;
+            }
+            mutableDict->addWord(word, wordLen, status, value.getLong());
+            if (U_FAILURE(status)) {
+                errln("Could not add word to mutable dictionary; status %s\n", u_errorName(status));
+                goto cleanup;
+            }
+            wordCount += 1;
+        }
+
+        // Find beginning of next line
+        while (uc && breaks->contains(uc)) {
+            uc = *current++;
+        }
+        word = current-1;
+        wordLen = 0;
+    }
+    
+    if (wordCount < 50) {
+        errln("Word count (%d) unreasonably small\n", wordCount);
+        goto cleanup;
+    }
+
+    enumer1 = mutableDict->openWords(status);
+    if (U_FAILURE(status)) {
+        errln("Could not open mutable dictionary enumerator: %s\n", u_errorName(status));
+        goto cleanup;
+    }
+
+    testCount = 0;
+    if (wordCount != (testCount = enumer1->count(status))) {
+        errln("MutableTrieDictionary word count (%d) differs from file word count (%d), with status %s\n",
+                testCount, wordCount, u_errorName(status));
+        goto cleanup;
+    }
+    
+    // Now compact it
+    compactDict = new CompactTrieDictionary(*mutableDict, status);
+    if (U_FAILURE(status)) {
+        errln("Failed to create CompactTrieDictionary: %s\n", u_errorName(status));
+        goto cleanup;
+    }
+    
+    enumer2 = compactDict->openWords(status);
+    if (U_FAILURE(status)) {
+        errln("Could not open compact trie dictionary enumerator: %s\n", u_errorName(status));
+        goto cleanup;
+    }
+
+
+    //delete later
+//    writeEnumerationToFile(enumer1, "/home/jchye/mutable.txt");
+//    writeEnumerationToFile(enumer2, "/home/jchye/compact.txt");
+
+    enumer1->reset(status);
+    enumer2->reset(status);
+
+    originalWord = enumer1->snext(status);
+    cloneWord = enumer2->snext(status);
+    while (U_SUCCESS(status) && originalWord != NULL && cloneWord != NULL) {
+        if (*originalWord != *cloneWord) {
+            errln("MutableTrieDictionary and CompactTrieDictionary word mismatch at %d, lengths are %d and %d\n", 
+                    counter, originalWord->length(), cloneWord->length());
+            goto cleanup;
+        }
+        
+        // check if attached values of the same word in both dictionaries tally
+#if 0
+        int32_t lengths1[originalWord->length()], lengths2[cloneWord->length()];
+        uint16_t values1[originalWord->length()], values2[cloneWord->length()];
+#endif
+        AutoBuffer<int32_t, 20> lengths1(originalWord->length());
+        AutoBuffer<int32_t, 20> lengths2(cloneWord->length());
+        AutoBuffer<uint16_t, 20> values1(originalWord->length());
+        AutoBuffer<uint16_t, 20> values2(cloneWord->length());
+      
+        originalText = utext_openConstUnicodeString(originalText, originalWord, &status);
+        cloneText = utext_openConstUnicodeString(cloneText, cloneWord, &status);
+        
+        int count1, count2;
+        mutableDict->matches(originalText, originalWord->length(), lengths1.elems(), count1, originalWord->length(), values1.elems());
+        compactDict->matches(cloneText, cloneWord->length(), lengths2.elems(), count2, cloneWord->length(), values2.elems());
+        
+        if(values1[count1-1] != values2[count2-1]){
+            errln("Values of word %d in MutableTrieDictionary and CompactTrieDictionary do not match, with values %d and %d\n", 
+                  counter, values1[count1-1], values2[count2-1]);
+            goto cleanup;
+        }
+        
+        counter++;
+        originalWord = enumer1->snext(status);
+        cloneWord = enumer2->snext(status);
+    }
+    if (enumer1->getDynamicClassID() == enumer2->getDynamicClassID()) {
+        errln("CompactTrieEnumeration and MutableTrieEnumeration ClassIDs are the same");
+    }
+    
+    delete enumer1;
+    enumer1 = NULL;
+    delete enumer2;
+    enumer2 = NULL;
+
+    // Now un-compact it
+    mutable2 = compactDict->cloneMutable(status);
+    if (U_FAILURE(status)) {
+        errln("Could not clone CompactTrieDictionary to MutableTrieDictionary: %s\n", u_errorName(status));
+        goto cleanup;
+    }
+
+    cloneEnum = mutable2->openWords(status);
+    if (U_FAILURE(status)) {
+        errln("Could not create cloned mutable enumerator: %s\n", u_errorName(status));
+        goto cleanup;
+    }
+
+    if (wordCount != (testCount = cloneEnum->count(status))) {
+        errln("Cloned MutableTrieDictionary word count (%d) differs from file word count (%d), with status %s\n",
+                testCount, wordCount, u_errorName(status));
+        goto cleanup;
+    }
+
+    // Compact original dictionary to clone. Note that we can only compare the same kind of
+    // dictionary as the order of the enumerators is not guaranteed to be the same between
+    // different kinds
+    enumer1 = mutableDict->openWords(status);
+    if (U_FAILURE(status)) {
+        errln("Could not re-open mutable dictionary enumerator: %s\n", u_errorName(status));
+        goto cleanup;
+    }
+
+    counter = 0;
+    originalWord = enumer1->snext(status);
+    cloneWord = cloneEnum->snext(status);
+    while (U_SUCCESS(status) && originalWord != NULL && cloneWord != NULL) {
+        if (*originalWord != *cloneWord) {
+            errln("Original and cloned MutableTrieDictionary word mismatch\n");
+            goto cleanup;
+        }
+
+        // check if attached values of the same word in both dictionaries tally
+        AutoBuffer<int32_t, 20> lengths1(originalWord->length());
+        AutoBuffer<int32_t, 20> lengths2(cloneWord->length());
+        AutoBuffer<uint16_t, 20> values1(originalWord->length());
+        AutoBuffer<uint16_t, 20> values2(cloneWord->length());
+        originalText = utext_openConstUnicodeString(originalText, originalWord, &status);
+        cloneText = utext_openConstUnicodeString(cloneText, cloneWord, &status);
+        
+        int count1, count2;
+        mutableDict->matches(originalText, originalWord->length(), lengths1.elems(), count1, originalWord->length(), values1.elems());
+        mutable2->matches(cloneText, cloneWord->length(), lengths2.elems(), count2, cloneWord->length(), values2.elems());
+        
+        if(values1[count1-1] != values2[count2-1]){
+            errln("Values of word %d in original and cloned MutableTrieDictionary do not match, with values %d and %d\n", 
+                  counter, values1[count1-1], values2[count2-1]);
+            goto cleanup;
+        }
+        
+        counter++;
+
+        originalWord = enumer1->snext(status);
+        cloneWord = cloneEnum->snext(status);
+    }
+
+    if (U_FAILURE(status)) {
+        errln("Enumeration failed: %s\n", u_errorName(status));
+        goto cleanup;
+    }
+
+    if (originalWord != cloneWord) {
+        errln("Original and cloned MutableTrieDictionary ended enumeration at different points\n");
+        goto cleanup;
+    }
+
+    // Test the data copying constructor for CompactTrieDict, and the data access APIs.
+    compact2 = new CompactTrieDictionary(compactDict->data(), status);
+    if (U_FAILURE(status)) {
+        errln("CompactTrieDictionary(const void *,...) failed\n");
+        goto cleanup;
+    }
+
+    if (compact2->dataSize() == 0) {
+        errln("CompactTrieDictionary->dataSize() == 0\n");
+        goto cleanup;
+    }
+
+    // Now count the words via the second dictionary
+    delete enumer1;
+    enumer1 = compact2->openWords(status);
+    if (U_FAILURE(status)) {
+        errln("Could not open compact trie dictionary 2 enumerator: %s\n", u_errorName(status));
+        goto cleanup;
+    }
+
+    if (wordCount != (testCount = enumer1->count(status))) {
+        errln("CompactTrieDictionary 2 word count (%d) differs from file word count (%d), with status %s\n",
+                testCount, wordCount, u_errorName(status));
+        goto cleanup;
+    }
+
+    cleanup:
+    delete compactDict;
+    delete mutableDict;
+    delete breaks;
+    delete[] testFile;
+    delete enumer1;
+    delete mutable2;
+    delete cloneEnum;
+    delete compact2;
+    utext_close(originalText);
+    utext_close(cloneText);
+
+ 
+}
 
 //----------------------------------------------------------------------------
 //
 // generalIteratorTest      Given a break iterator and a set of test data,
 //                          Run the tests and report the results.
 //
 //----------------------------------------------------------------------------
 void RBBITest::generalIteratorTest(RuleBasedBreakIterator& bi, BITestData &td)
 {
 
@@ skipping 1001 matching lines @@
 // UBreakIteratorType UBRK_WORD, Locale "en_US_POSIX"
 // Words don't include colon or period (cldrbug #1969).
 static const char    posxWordText[]     = "Can't have breaks in xx:yy or struct.field for CS-types.";
 static const int32_t posxWordTOffsets[] = { 5, 6, 10, 11, 17, 18, 20, 21, 23, 24, 26, 27, 29, 30, 36, 37, 42, 43, 46, 47, 49, 50, 55, 56 };
 static const int32_t posxWordROffsets[] = { 5, 6, 10, 11, 17, 18, 20, 21,         26, 27, 29, 30,         42, 43, 46, 47, 49, 50, 55, 56 };
 
 // UBreakIteratorType UBRK_WORD, Locale "ja"
 // Don't break in runs of hiragana or runs of ideograph, where the latter includes \u3005 \u3007 \u303B (cldrbug #2009).
 static const char    jaWordText[]     = "\\u79C1\\u9054\\u306B\\u4E00\\u3007\\u3007\\u3007\\u306E\\u30B3\\u30F3\\u30D4\\u30E5\\u30FC\\u30BF"
                                         "\\u304C\\u3042\\u308B\\u3002\\u5948\\u3005\\u306F\\u30EF\\u30FC\\u30C9\\u3067\\u3042\\u308B\\u3002";
+#if 0 
 static const int32_t jaWordTOffsets[] = {    2, 3,          7, 8, 14,         17, 18,     20, 21, 24,         27, 28 };
 static const int32_t jaWordROffsets[] = { 1, 2, 3, 4, 5, 6, 7, 8, 14, 15, 16, 17, 18, 19, 20, 21, 24, 25, 26, 27, 28 };
+#endif
+// There's no separate Japanese word break iterator. Root is the same as Japanese.
+// Our dictionary-based iterator has to be tweaked to better handle U+3005,
+// U+3007, U+300B and some other cases.
+static const int32_t jaWordTOffsets[] = { 1, 2, 3, 4, 5,    7, 8, 12, 13, 14, 15, 17, 18, 20, 21, 22, 23, 24, 25, 27, 28 };
+static const int32_t jaWordROffsets[] = { 1, 2, 3, 4, 5,    7, 8, 12, 13, 14, 15, 17, 18, 20, 21, 22, 23, 24, 25, 27, 28 };
 
 // UBreakIteratorType UBRK_SENTENCE, Locale "el"
 // Add break after Greek question mark (cldrbug #2069).
 static const char    elSentText[]     = "\\u0391\\u03B2, \\u03B3\\u03B4; \\u0395 \\u03B6\\u03B7\\u037E \\u0398 \\u03B9\\u03BA. "
                                         "\\u039B\\u03BC \\u03BD\\u03BE! \\u039F\\u03C0, \\u03A1\\u03C2? \\u03A3";
 static const int32_t elSentTOffsets[] = { 8, 14, 20, 27, 35, 36 };
 static const int32_t elSentROffsets[] = {        20, 27, 35, 36 };
 
 // UBreakIteratorType UBRK_CHARACTER, Locale "th"
 // Clusters should not include spacing Thai/Lao vowels (prefix or postfix), except for [SARA] AM (cldrbug #2161).
@@ skipping 780 matching lines @@
     virtual  void     setText(const UnicodeString &s);
     virtual int32_t   next(int32_t i);
 private:
     UVector      *fSets;
 
     UnicodeSet  *fCRSet;
     UnicodeSet  *fLFSet;
     UnicodeSet  *fNewlineSet;
     UnicodeSet  *fKatakanaSet;
     UnicodeSet  *fALetterSet;
+    // TODO(jungshik): Do we still need this change? 
+    // UnicodeSet  *fALetterSet; // matches ALetterPlus in word.txt
     UnicodeSet  *fMidNumLetSet;
     UnicodeSet  *fMidLetterSet;
     UnicodeSet  *fMidNumSet;
     UnicodeSet  *fNumericSet;
     UnicodeSet  *fFormatSet;
     UnicodeSet  *fOtherSet;
     UnicodeSet  *fExtendSet;
     UnicodeSet  *fExtendNumLetSet;
+    UnicodeSet  *fDictionaryCjkSet;
 
     RegexMatcher  *fMatcher;
 
     const UnicodeString  *fText;
 };
 
 
 RBBIWordMonkey::RBBIWordMonkey()
 {
     UErrorCode  status = U_ZERO_ERROR;
 
     fSets            = new UVector(status);
 
     fCRSet           = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = CR}]"),           status);
     fLFSet           = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = LF}]"),           status);
     fNewlineSet      = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = Newline}]"),      status);
-    fALetterSet      = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = ALetter}]"),      status);
+    fDictionaryCjkSet= new UnicodeSet("[[\\uac00-\\ud7a3][:Han:][:Hiragana:]]", status);
+    // Exclude Hangul syllables from ALetterSet during testing.
+    // Leave CJK dictionary characters out from the monkey tests!
+#if 0 
+    fALetterSet      = new UnicodeSet("[\\p{Word_Break = ALetter}"
+                                      "[\\p{Line_Break = Complex_Context}"
+                                      "-\\p{Grapheme_Cluster_Break = Extend}"
+                                      "-\\p{Grapheme_Cluster_Break = Control}"
+                                      "]]",
+                                      status);
+#endif
+    fALetterSet      = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = ALetter}]"), status);
+    fALetterSet->removeAll(*fDictionaryCjkSet);
     fKatakanaSet     = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = Katakana}]"),     status);
     fMidNumLetSet    = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = MidNumLet}]"),    status);
     fMidLetterSet    = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = MidLetter}]"),    status);
     fMidNumSet       = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = MidNum}]"),       status);
-    fNumericSet      = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = Numeric}]"),      status);
+    fNumericSet      = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = Numeric}[\\uff10-\\uff19]]"),      status);
     fFormatSet       = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = Format}]"),       status);
     fExtendNumLetSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = ExtendNumLet}]"), status);
     fExtendSet       = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = Extend}]"),       status);
 
     fOtherSet        = new UnicodeSet();
     if(U_FAILURE(status)) {
       deferredStatus = status;
       return;
     }
 
     fOtherSet->complement();
     fOtherSet->removeAll(*fCRSet);
     fOtherSet->removeAll(*fLFSet);
     fOtherSet->removeAll(*fNewlineSet);
     fOtherSet->removeAll(*fKatakanaSet);
     fOtherSet->removeAll(*fALetterSet);
     fOtherSet->removeAll(*fMidLetterSet);
     fOtherSet->removeAll(*fMidNumSet);
     fOtherSet->removeAll(*fNumericSet);
     fOtherSet->removeAll(*fExtendNumLetSet);
     fOtherSet->removeAll(*fFormatSet);
     fOtherSet->removeAll(*fExtendSet);
     // Inhibit dictionary characters from being tested at all.
+    fOtherSet->removeAll(*fDictionaryCjkSet);
     fOtherSet->removeAll(UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{LineBreak = Complex_Context}]"), status));
 
     fSets->addElement(fCRSet,        status);
     fSets->addElement(fLFSet,        status);
     fSets->addElement(fNewlineSet,   status);
     fSets->addElement(fALetterSet,   status);
-    fSets->addElement(fKatakanaSet,  status);
+    //fSets->addElement(fKatakanaSet,  status); //TODO: work out how to test katakana
     fSets->addElement(fMidLetterSet, status);
     fSets->addElement(fMidNumLetSet, status);
     fSets->addElement(fMidNumSet,    status);
     fSets->addElement(fNumericSet,   status);
     fSets->addElement(fFormatSet,    status);
     fSets->addElement(fExtendSet,    status);
     fSets->addElement(fOtherSet,     status);
     fSets->addElement(fExtendNumLetSet, status);
 
     if (U_FAILURE(status)) {
@@ skipping 1226 matching lines @@
                 printStringBreaks(ustr, expected, expectedcount);
                 test->errln("isBoundary() failed.  Not expecting boundary at position %d", j);
                 return;
             }
         }
     }
 
     for (i = bi->last(); i != BreakIterator::DONE; i = bi->previous()) {
         count --;
         if (forward[count] != i) {
+            printStringBreaks(ustr, expected, expectedcount);
             test->errln("happy break test previous() failed: expected %d but got %d",
                         forward[count], i);
             break;
         }
     }
     if (count != 0) {
         printStringBreaks(ustr, expected, expectedcount);
         test->errln("break test previous() failed: missed a match");
         return;
     }
@@ skipping 13 matching lines @@
 }
 
 void RBBITest::TestWordBreaks(void)
 {
 #if !UCONFIG_NO_REGULAR_EXPRESSIONS
 
     Locale        locale("en");
     UErrorCode    status = U_ZERO_ERROR;
     // BreakIterator  *bi = BreakIterator::createCharacterInstance(locale, status);
     BreakIterator *bi = BreakIterator::createWordInstance(locale, status);
+    // Replaced any C+J characters in a row with a random sequence of characters
+    // of the same length to make our C+J segmentation not get in the way.
     static const char *strlist[] =
     {
     "\\U000e0032\\u0097\\u0f94\\uc2d8\\u05f4\\U000e0031\\u060d",
-    "\\U000e0037\\u4666\\u1202\\u003a\\U000e0031\\u064d\\u0bea\\u591c\\U000e0040\\u003b",
+    "\\U000e0037\\u2666\\u1202\\u003a\\U000e0031\\u064d\\u0bea\\u091c\\U000e0040\\u003b",
     "\\u0589\\u3e99\\U0001d7f3\\U000e0074\\u1810\\u200e\\U000e004b\\u0027\\U000e0061\\u003a",
     "\\u398c\\U000104a5\\U0001d173\\u102d\\u002e\\uca3b\\u002e\\u002c\\u5622",
-    "\\u90ca\\u3588\\u009c\\u0953\\u194b",
+    "\\uac00\\u3588\\u009c\\u0953\\u194b",
     "\\u200e\\U000e0072\\u0a4b\\U000e003f\\ufd2b\\u2027\\u002e\\u002e",
     "\\u0602\\u2019\\ua191\\U000e0063\\u0a4c\\u003a\\ub4b5\\u003a\\u827f\\u002e",
-    "\\u7f1f\\uc634\\u65f8\\u0944\\u04f2\\uacdf\\u1f9c\\u05f4\\u002e",
+    "\\u2f1f\\u1634\\u05f8\\u0944\\u04f2\\u0cdf\\u1f9c\\u05f4\\u002e",
     "\\U000e0042\\u002e\\u0fb8\\u09ef\\u0ed1\\u2044",
     "\\u003b\\u024a\\u102e\\U000e0071\\u0600",
     "\\u2027\\U000e0067\\u0a47\\u00b7",
     "\\u1fcd\\u002c\\u07aa\\u0027\\u11b0",
     "\\u002c\\U000e003c\\U0001d7f4\\u003a\\u0c6f\\u0027",
     "\\u0589\\U000e006e\\u0a42\\U000104a5",
-    "\\u4f66\\ub523\\u003a\\uacae\\U000e0047\\u003a",
+    "\\u0f66\\u2523\\u003a\\u0cae\\U000e0047\\u003a",
     "\\u003a\\u0f21\\u0668\\u0dab\\u003a\\u0655\\u00b7",
     "\\u0027\\u11af\\U000e0057\\u0602",
     "\\U0001d7f2\\U000e007\\u0004\\u0589",
     "\\U000e0022\\u003a\\u10b3\\u003a\\ua21b\\u002e\\U000e0058\\u1732\\U000e002b",
     "\\U0001d7f2\\U000e007d\\u0004\\u0589",
     "\\u82ab\\u17e8\\u0736\\u2019\\U0001d64d",
     "\\ub55c\\u0a68\\U000e0037\\u0cd6\\u002c\\ub959",
     "\\U000e0065\\u302c\\uc986\\u09ee\\U000e0068",
     "\\u0be8\\u002e\\u0c68\\u066e\\u136d\\ufc99\\u59e7",
     "\\u0233\\U000e0020\\u0a69\\u0d6a",
     "\\u206f\\u0741\\ub3ab\\u2019\\ubcac\\u2019",
-    "\\u58f4\\U000e0049\\u20e7\\u2027",
+    "\\u18f4\\U000e0049\\u20e7\\u2027",
     "\\ub315\\U0001d7e5\\U000e0073\\u0c47\\u06f2\\u0c6a\\u0037\\u10fe",
     "\\ua183\\u102d\\u0bec\\u003a",
     "\\u17e8\\u06e7\\u002e\\u096d\\u003b",
     "\\u003a\\u0e57\\u0fad\\u002e",
     "\\u002e\\U000e004c\\U0001d7ea\\u05bb\\ud0fd\\u02de",
     "\\u32e6\\U0001d7f6\\u0fa1\\u206a\\U000e003c\\u0cec\\u003a",
     "\\U000e005d\\u2044\\u0731\\u0650\\u0061",
     "\\u003a\\u0664\\u00b7\\u1fba",
     "\\u003b\\u0027\\u00b7\\u47a3",
-    "\\u2027\\U000e0067\\u0a42\\u00b7\\ubddf\\uc26c\\u003a\\u4186\\u041b",
+    "\\u2027\\U000e0067\\u0a42\\u00b7\\u4edf\\uc26c\\u003a\\u4186\\u041b",
     "\\u0027\\u003a\\U0001d70f\\U0001d7df\\ubf4a\\U0001d7f5\\U0001d177\\u003a\\u0e51\\u1058\\U000e0058\\u00b7\\u0673",
     "\\uc30d\\u002e\\U000e002c\\u0c48\\u003a\\ub5a1\\u0661\\u002c",
     };
     int loop;
     if (U_FAILURE(status)) {
         errcheckln(status, "Creation of break iterator failed %s", u_errorName(status));
         return;
     }
     for (loop = 0; loop < (int)(sizeof(strlist) / sizeof(char *)); loop ++) {
         // printf("looping %d\n", loop);
@@ skipping 34 matching lines @@
     "\\u002c\\U000e003c\\U0001d7f4\\u003a\\u0c6f\\u0027",
     "\\u0589\\U000e006e\\u0a42\\U000104a5",
     "\\u4f66\\ub523\\u003a\\uacae\\U000e0047\\u003a",
     "\\u003a\\u0f21\\u0668\\u0dab\\u003a\\u0655\\u00b7",
     "\\u0027\\u11af\\U000e0057\\u0602",
     "\\U0001d7f2\\U000e007\\u0004\\u0589",
     "\\U000e0022\\u003a\\u10b3\\u003a\\ua21b\\u002e\\U000e0058\\u1732\\U000e002b",
     "\\U0001d7f2\\U000e007d\\u0004\\u0589",
     "\\u82ab\\u17e8\\u0736\\u2019\\U0001d64d",
     "\\u0e01\\ub55c\\u0a68\\U000e0037\\u0cd6\\u002c\\ub959",
-    "\\U000e0065\\u302c\\uc986\\u09ee\\U000e0068",
+    "\\U000e0065\\u302c\\u09ee\\U000e0068",
     "\\u0be8\\u002e\\u0c68\\u066e\\u136d\\ufc99\\u59e7",
     "\\u0233\\U000e0020\\u0a69\\u0d6a",
     "\\u206f\\u0741\\ub3ab\\u2019\\ubcac\\u2019",
     "\\u58f4\\U000e0049\\u20e7\\u2027",
-    "\\ub315\\U0001d7e5\\U000e0073\\u0c47\\u06f2\\u0c6a\\u0037\\u10fe",
+    "\\U0001d7e5\\U000e0073\\u0c47\\u06f2\\u0c6a\\u0037\\u10fe",
     "\\ua183\\u102d\\u0bec\\u003a",
     "\\u17e8\\u06e7\\u002e\\u096d\\u003b",
     "\\u003a\\u0e57\\u0fad\\u002e",
     "\\u002e\\U000e004c\\U0001d7ea\\u05bb\\ud0fd\\u02de",
     "\\u32e6\\U0001d7f6\\u0fa1\\u206a\\U000e003c\\u0cec\\u003a",
     "\\ua2a5\\u0038\\u2044\\u002e\\u0c67\\U000e003c\\u05f4\\u2027\\u05f4\\u2019",
     "\\u003a\\u0664\\u00b7\\u1fba",
     "\\u003b\\u0027\\u00b7\\u47a3",
     };
     int loop;
@@ skipping 662 matching lines @@
     pos = bi->last();
     do {
         // ruleStatus = bi->getRuleStatus();
         printf("%d\t%d\n", pos, ruleStatus);
         pos = bi->previous();
     } while (pos != BreakIterator::DONE);
 #endif
 }
 
 #endif /* #if !UCONFIG_NO_BREAK_ITERATION */