CJK segmentation patch for ICU 4.6...

icu46/source/common/brkeng.cpp

 /**
  ************************************************************************************
  * Copyright (C) 2006-2009, International Business Machines Corporation and others. *
  * All Rights Reserved.                                                             *
  ************************************************************************************
  */
 
 #include "unicode/utypes.h"
 
 #if !UCONFIG_NO_BREAK_ITERATION
@@ skipping 208 matching lines @@
     UErrorCode status = U_ZERO_ERROR;
     UScriptCode code = uscript_getScript(c, &status);
     if (U_SUCCESS(status)) {
         const CompactTrieDictionary *dict = loadDictionaryFor(code, breakType);
         if (dict != NULL) {
             const LanguageBreakEngine *engine = NULL;
             switch(code) {
             case USCRIPT_THAI:
                 engine = new ThaiBreakEngine(dict, status);
                 break;
+                
+            case USCRIPT_HANGUL:
+                engine = new CjkBreakEngine(dict, kKorean, status);
+                break;
+
+            // use same BreakEngine and dictionary for both Chinese and Japanese
+            case USCRIPT_HIRAGANA:
+            case USCRIPT_KATAKANA:
+            case USCRIPT_HAN:
+                engine = new CjkBreakEngine(dict, kChineseJapanese, status);
+                break;
+#if 0
+            // TODO: Have to get some characters with script=common handled
+            // by CjkBreakEngine (e.g. U+309B). Simply subjecting
+            // them to CjkBreakEngine does not work. The engine has to
+            // special-case them.
+            case USCRIPT_COMMON:
+            {
+                UBlockCode block = ublock_getCode(code);
+                if (block == UBLOCK_HIRAGANA || block == UBLOCK_KATAKANA)
+                   engine = new CjkBreakEngine(dict, kChineseJapanese, status);
+                break;
+            }
+#endif
             default:
                 break;
             }
             if (engine == NULL) {
                 delete dict;
             }
             else if (U_FAILURE(status)) {
                 delete engine;
                 engine = NULL;
             }
@@ skipping 35 matching lines @@
         const CompactTrieDictionary *dict = new CompactTrieDictionary(
             file, status);
         if (U_SUCCESS(status) && dict == NULL) {
             status = U_MEMORY_ALLOCATION_ERROR;
         }
         if (U_FAILURE(status)) {
             delete dict;
             dict = NULL;
         }
         return dict;
+    } else if (dictfname != NULL){
+        //create dummy dict if dictionary filename not valid
+        UChar c = 0x0020;
+        status = U_ZERO_ERROR;
+        MutableTrieDictionary *mtd = new MutableTrieDictionary(c, status, TRUE);
+        mtd->addWord(&c, 1, status, 1);
+        return new CompactTrieDictionary(*mtd, status);  
     }
     return NULL;
 }
 
 U_NAMESPACE_END
 
 #endif /* #if !UCONFIG_NO_BREAK_ITERATION */