CJK segmentation patch for ICU 4.6...

icu46/source/test/intltest/rbbiapts.cpp

 /********************************************************************
  * Copyright (c) 1999-2010, International Business Machines
  * Corporation and others. All Rights Reserved.
  ********************************************************************
  *   Date        Name        Description
  *   12/14/99    Madhu        Creation.
  *   01/12/2000  Madhu        updated for changed API
  ********************************************************************/
 
 #include "unicode/utypes.h"
@@ skipping 138 matching lines @@
     UErrorCode status = U_ZERO_ERROR;
     BreakIterator* a = BreakIterator::createWordInstance(Locale("hi"), status);
     BreakIterator* b = BreakIterator::createWordInstance(Locale("hi_IN"),status);
     if (U_FAILURE(status)) {
         errcheckln(status, "Creation of break iterator failed %s", u_errorName(status));
         return;
     }
     if(*a!=*b){
         errln("Failed: boilerplate method operator!= does not return correct results");
     }
-    BreakIterator* c = BreakIterator::createWordInstance(Locale("ja"),status);
-    if(a && c){
-        if(*c==*a){
+    // Japanese word break iteratos is identical to root with
+    // a dictionary-based break iterator, but Thai character break iterator
+    // is still different from Root. 
+    BreakIterator* c = BreakIterator::createCharacterInstance(Locale("ja"),status);
+    BreakIterator* d = BreakIterator::createCharacterInstance(Locale("th"),status);
+    if(c && d){
+        if(*c==*d){
             errln("Failed: boilerplate method opertator== does not return correct results");
         }
     }else{
         errln("creation of break iterator failed");
     }
     delete a;
     delete b;
     delete c;
+    delete d;
 }
 
 void RBBIAPITest::TestgetRules()
 {
     UErrorCode status=U_ZERO_ERROR;
 
     RuleBasedBreakIterator* bi1=(RuleBasedBreakIterator*)RuleBasedBreakIterator::createCharacterInstance(Locale::getDefault(), status);
     RuleBasedBreakIterator* bi2=(RuleBasedBreakIterator*)RuleBasedBreakIterator::createWordInstance(Locale::getDefault(), status);
     if(U_FAILURE(status)){
         errcheckln(status, "FAIL: in construction - %s", u_errorName(status));
@@ skipping 448 matching lines @@
      }
      delete bi;
 }
 
 //
 //  TestRuleStatus
 //      Test word break rule status constants.
 //
 void RBBIAPITest::TestRuleStatus() {
      UChar str[30];
-     u_unescape("plain word 123.45 \\u9160\\u9161 \\u30a1\\u30a2 \\u3041\\u3094",
-              // 012345678901234567  8      9    0  1      2    3  4      5    6
-              //                    Ideographic    Katakana       Hiragana
+     //no longer test Han or hiragana breaking here: ruleStatusVec would return nothing
+     // changed UBRK_WORD_KANA to UBRK_WORD_IDEO
+     u_unescape("plain word 123.45 \\u30a1\\u30a2 ",
+              // 012345678901234567  8      9    0      
+              //                     Katakana      
                 str, 30);
      UnicodeString testString1(str);
-     int32_t bounds1[] = {0, 5, 6, 10, 11, 17, 18, 19, 20, 21, 23, 24, 25, 26};
+     int32_t bounds1[] = {0, 5, 6, 10, 11, 17, 18, 20, 21};
      int32_t tag_lo[]  = {UBRK_WORD_NONE,     UBRK_WORD_LETTER, UBRK_WORD_NONE,    UBRK_WORD_LETTER,
                           UBRK_WORD_NONE,     UBRK_WORD_NUMBER, UBRK_WORD_NONE,
-                          UBRK_WORD_IDEO,     UBRK_WORD_IDEO,   UBRK_WORD_NONE,
-                          UBRK_WORD_KANA,     UBRK_WORD_NONE,   UBRK_WORD_KANA,    UBRK_WORD_KANA};
+                          UBRK_WORD_IDEO,     UBRK_WORD_NONE};
 
      int32_t tag_hi[]  = {UBRK_WORD_NONE_LIMIT, UBRK_WORD_LETTER_LIMIT, UBRK_WORD_NONE_LIMIT, UBRK_WORD_LETTER_LIMIT,
                           UBRK_WORD_NONE_LIMIT, UBRK_WORD_NUMBER_LIMIT, UBRK_WORD_NONE_LIMIT,
-                          UBRK_WORD_IDEO_LIMIT, UBRK_WORD_IDEO_LIMIT,   UBRK_WORD_NONE_LIMIT,
-                          UBRK_WORD_KANA_LIMIT, UBRK_WORD_NONE_LIMIT,   UBRK_WORD_KANA_LIMIT, UBRK_WORD_KANA_LIMIT};
+                          UBRK_WORD_IDEO_LIMIT, UBRK_WORD_NONE_LIMIT};
 
      UErrorCode status=U_ZERO_ERROR;
 
      RuleBasedBreakIterator *bi = (RuleBasedBreakIterator *)BreakIterator::createWordInstance(Locale::getEnglish(), status);
      if(U_FAILURE(status)) {
          errcheckln(status, "Fail : in construction - %s", u_errorName(status));
      } else {
          bi->setText(testString1);
          // First test that the breaks are in the right spots.
          doBoundaryTest(*bi, testString1, bounds1);
@@ skipping 218 matching lines @@
         delete ja_word;
         delete ja_char;
         delete root_word;
         delete root_char;
         
         return;
     }
 
     URegistryKey key = BreakIterator::registerInstance(ja_word, "xx", UBRK_WORD, status);
     {
+#if 0 // With a dictionary based word breaking, ja_word is identical to root.
         if (ja_word && *ja_word == *root_word) {
             errln("japan not different from root");
         }
+#endif
     }
 
     {
         BreakIterator* result = BreakIterator::createWordInstance("xx_XX", status);
         UBool fail = TRUE;
         if(result){
             fail = *result != *ja_word;
         }
         delete result;
         if (fail) {
@@ skipping 273 matching lines @@
     : RuleBasedBreakIterator(data, status)
 {
 }
 
 RBBIWithProtectedFunctions::RBBIWithProtectedFunctions(const RBBIDataHeader* data, enum EDontAdopt, UErrorCode &status)
     : RuleBasedBreakIterator(data, RuleBasedBreakIterator::kDontAdopt, status)
 {
 }
 
 #endif /* #if !UCONFIG_NO_BREAK_ITERATION */