CJK segmentation patch for ICU 4.6...

icu46/source/common/rbbi.cpp

 /*
 ***************************************************************************
 *   Copyright (C) 1999-2010 International Business Machines Corporation
 *   and others. All rights reserved.
 ***************************************************************************
 */
 //
 //  file:  rbbi.c    Contains the implementation of the rule based break iterator
 //                   runtime engine and the API implementation for
 //                   class RuleBasedBreakIterator
@@ skipping 1537 matching lines @@
 //  checkDictionary       This function handles all processing of characters in
 //                        the "dictionary" set. It will determine the appropriate
 //                        course of action, and possibly set up a cache in the
 //                        process.
 //
 //-------------------------------------------------------------------------------
 int32_t RuleBasedBreakIterator::checkDictionary(int32_t startPos,
                             int32_t endPos,
                             UBool reverse) {
     // Reset the old break cache first.
-    uint32_t dictionaryCount = fDictionaryCharCount;
     reset();
 
-    if (dictionaryCount <= 1 || (endPos - startPos) <= 1) {
+    // note: code segment below assumes that dictionary chars are in the 
+    // startPos-endPos range
+    // value returned should be next character in sequence
+    if ((endPos - startPos) <= 1) {
         return (reverse ? startPos : endPos);
     }
     
     // Bug 5532.  The dictionary code will crash if the input text is UTF-8
     //      because native indexes are different from UTF-16 indexes.
     //      Temporary hack: skip dictionary lookup for UTF-8 encoded text.
     //      It wont give the right breaks, but it's better than a crash.
     //
     //      Check the type of the UText by checking its pFuncs field, which
     //      is UText's function dispatch table.  It will be the same for all
@@ skipping 132 matching lines @@
             for (int32_t i = 0; i < foundBreakCount; ++i) {
                 fCachedBreakPositions[out++] = breaks.elementAti(i);
             }
             if (endPos > fCachedBreakPositions[out-1]) {
                 fCachedBreakPositions[out] = endPos;
             }
             // If there are breaks, then by definition, we are replacing the original
             // proposed break by one of the breaks we found. Use following() and
             // preceding() to do the work. They should never recurse in this case.
             if (reverse) {
-                return preceding(endPos - 1);
+                return preceding(endPos);
             }
             else {
                 return following(startPos);
             }
         }
         // If the allocation failed, just fall through to the "no breaks found" case.
     }
 
     // If we get here, there were no language-based breaks. Set the text pointer
     // to the original proposed break.
@@ skipping 145 matching lines @@
 }*/
 
 void RuleBasedBreakIterator::setBreakType(int32_t type) {
     fBreakType = type;
     reset();
 }
 
 U_NAMESPACE_END
 
 #endif /* #if !UCONFIG_NO_BREAK_ITERATION */