Update ICU to 54.1 step 1

source/common/rbbi.cpp

 /*
 ***************************************************************************
-*   Copyright (C) 1999-2013 International Business Machines Corporation
+*   Copyright (C) 1999-2014 International Business Machines Corporation
 *   and others. All rights reserved.
 ***************************************************************************
 */
 //
 //  file:  rbbi.c    Contains the implementation of the rule based break iterator
 //                   runtime engine and the API implementation for
 //                   class RuleBasedBreakIterator
 //
 
 #include "utypeinfo.h"  // for 'typeid' to work
@@ skipping 497 matching lines @@
         // contents as the old.  If we can't set to the same position, it doesn't.
         // The contents underlying the old utext might be invalid at this point,
         // so it's not safe to check directly.
         status = U_ILLEGAL_ARGUMENT_ERROR;
     }
     return *this;
 }
 
 
 /**
- * Sets the current iteration position to the beginning of the text.
- * @return The offset of the beginning of the text.
+ * Sets the current iteration position to the beginning of the text, position zero.
+ * @return The new iterator position, which is zero.
  */
 int32_t RuleBasedBreakIterator::first(void) {
     reset();
     fLastRuleStatusIndex  = 0;
     fLastStatusIndexValid = TRUE;
     //if (fText == NULL)
     //    return BreakIterator::DONE;
 
     utext_setNativeIndex(fText, 0);
     return 0;
@@ skipping 52 matching lines @@
             int32_t pos = fCachedBreakPositions[fPositionInCache];
             utext_setNativeIndex(fText, pos);
             return pos;
         }
         else {
             reset();
         }
     }
 
     int32_t startPos = current();
+    fDictionaryCharCount = 0;
     int32_t result = handleNext(fData->fForwardTable);
     if (fDictionaryCharCount > 0) {
         result = checkDictionary(startPos, result, FALSE);
     }
     return result;
 }
 
 /**
  * Advances the iterator backwards, to the last boundary preceding this one.
  * @return The position of the last boundary position preceding this one.
@@ skipping 34 matching lines @@
             result = checkDictionary(result, startPos, TRUE);
         }
         return result;
     }
 
     // old rule syntax
     // set things up.  handlePrevious() will back us up to some valid
     // break position before the current position (we back our internal
     // iterator up one step to prevent handlePrevious() from returning
     // the current position), but not necessarily the last one before
-
     // where we started
 
     int32_t start = current();
 
     (void)UTEXT_PREVIOUS32(fText);
     int32_t lastResult    = handlePrevious(fData->fReverseTable);
     if (lastResult == UBRK_DONE) {
         lastResult = 0;
         utext_setNativeIndex(fText, 0);
     }
@@ skipping 12 matching lines @@
         }
         lastResult     = result;
         lastTag        = fLastRuleStatusIndex;
         breakTagValid  = TRUE;
     }
 
     // fLastBreakTag wants to have the value for section of text preceding
     // the result position that we are to return (in lastResult.)  If
     // the backwards rules overshot and the above loop had to do two or more
     // next()s to move up to the desired return position, we will have a valid
-    // tag value. But, if handlePrevious() took us to exactly the correct result positon,
+    // tag value. But, if handlePrevious() took us to exactly the correct result position,
     // we wont have a tag value for that position, which is only set by handleNext().
 
-    // set the current iteration position to be the last break position
-    // before where we started, and then return that value
+    // Set the current iteration position to be the last break position
+    // before where we started, and then return that value.
     utext_setNativeIndex(fText, lastResult);
     fLastRuleStatusIndex  = lastTag;       // for use by getRuleStatus()
     fLastStatusIndexValid = breakTagValid;
 
     // No need to check the dictionary; it will have been handled by
     // next()
 
     return lastResult;
 }
 
 /**
  * Sets the iterator to refer to the first boundary position following
  * the specified position.
  * @offset The position from which to begin searching for a break position.
  * @return The position of the first break after the current position.
  */
 int32_t RuleBasedBreakIterator::following(int32_t offset) {
+    // if the offset passed in is already past the end of the text,
+    // just return DONE; if it's before the beginning, return the
+    // text's starting offset
+    if (fText == NULL || offset >= utext_nativeLength(fText)) {
+        last();
+        return next();
+    }
+    else if (offset < 0) {
+        return first();
+    }
+
+    // Move requested offset to a code point start. It might be on a trail surrogate,
+    // or on a trail byte if the input is UTF-8.
+    utext_setNativeIndex(fText, offset);
+    offset = utext_getNativeIndex(fText);
+
     // if we have cached break positions and offset is in the range
     // covered by them, use them
     // TODO: could use binary search
     // TODO: what if offset is outside range, but break is not?
     if (fCachedBreakPositions != NULL) {
         if (offset >= fCachedBreakPositions[0]
                 && offset < fCachedBreakPositions[fNumCachedBreakPositions - 1]) {
             fPositionInCache = 0;
             // We are guaranteed not to leave the array due to range test above
             while (offset >= fCachedBreakPositions[fPositionInCache]) {
                 ++fPositionInCache;
             }
             int32_t pos = fCachedBreakPositions[fPositionInCache];
             utext_setNativeIndex(fText, pos);
             return pos;
         }
         else {
             reset();
         }
     }
 
-    // if the offset passed in is already past the end of the text,
-    // just return DONE; if it's before the beginning, return the
-    // text's starting offset
-    fLastRuleStatusIndex  = 0;
-    fLastStatusIndexValid = TRUE;
-    if (fText == NULL || offset >= utext_nativeLength(fText)) {
-        last();
-        return next();
-    }
-    else if (offset < 0) {
-        return first();
-    }
-
-    // otherwise, set our internal iteration position (temporarily)
+    // Set our internal iteration position (temporarily)
     // to the position passed in.  If this is the _beginning_ position,
     // then we can just use next() to get our return value
 
     int32_t result = 0;
 
     if (fData->fSafeRevTable != NULL) {
         // new rule syntax
         utext_setNativeIndex(fText, offset);
         // move forward one codepoint to prepare for moving back to a
         // safe point.
         // this handles offset being between a supplementary character
+        // TODO: is this still needed, with move to code point boundary handled above?
         (void)UTEXT_NEXT32(fText);
         // handlePrevious will move most of the time to < 1 boundary away
         handlePrevious(fData->fSafeRevTable);
         int32_t result = next();
         while (result <= offset) {
             result = next();
         }
         return result;
     }
     if (fData->fSafeFwdTable != NULL) {
@@ skipping 42 matching lines @@
     return result;
 }
 
 /**
  * Sets the iterator to refer to the last boundary position before the
  * specified position.
  * @offset The position to begin searching for a break from.
  * @return The position of the last boundary before the starting position.
  */
 int32_t RuleBasedBreakIterator::preceding(int32_t offset) {
+    // if the offset passed in is already past the end of the text,
+    // just return DONE; if it's before the beginning, return the
+    // text's starting offset
+    if (fText == NULL || offset > utext_nativeLength(fText)) {
+        return last();
+    }
+    else if (offset < 0) {
+        return first();
+    }
+
+    // Move requested offset to a code point start. It might be on a trail surrogate,
+    // or on a trail byte if the input is UTF-8.
+    utext_setNativeIndex(fText, offset);
+    offset = utext_getNativeIndex(fText);
+
     // if we have cached break positions and offset is in the range
     // covered by them, use them
     if (fCachedBreakPositions != NULL) {
         // TODO: binary search?
         // TODO: What if offset is outside range, but break is not?
         if (offset > fCachedBreakPositions[0]
                 && offset <= fCachedBreakPositions[fNumCachedBreakPositions - 1]) {
             fPositionInCache = 0;
             while (fPositionInCache < fNumCachedBreakPositions
                    && offset > fCachedBreakPositions[fPositionInCache])
                 ++fPositionInCache;
             --fPositionInCache;
             // If we're at the beginning of the cache, need to reevaluate the
             // rule status
             if (fPositionInCache <= 0) {
                 fLastStatusIndexValid = FALSE;
             }
             utext_setNativeIndex(fText, fCachedBreakPositions[fPositionInCache]);
             return fCachedBreakPositions[fPositionInCache];
         }
         else {
             reset();
         }
     }
 
-    // if the offset passed in is already past the end of the text,
-    // just return DONE; if it's before the beginning, return the
-    // text's starting offset
-    if (fText == NULL || offset > utext_nativeLength(fText)) {
-        // return BreakIterator::DONE;
-        return last();
-    }
-    else if (offset < 0) {
-        return first();
-    }
-
     // if we start by updating the current iteration position to the
     // position specified by the caller, we can just use previous()
     // to carry out this operation
 
     if (fData->fSafeFwdTable != NULL) {
         // new rule syntax
         utext_setNativeIndex(fText, offset);
         int32_t newOffset = (int32_t)UTEXT_GETNATIVEINDEX(fText);
         if (newOffset != offset) {
             // Will come here if specified offset was not a code point boundary AND
@@ skipping 713 matching lines @@
     // Reset the old break cache first.
     reset();
 
     // note: code segment below assumes that dictionary chars are in the 
     // startPos-endPos range
     // value returned should be next character in sequence
     if ((endPos - startPos) <= 1) {
         return (reverse ? startPos : endPos);
     }
     
-    // Bug 5532.  The dictionary code will crash if the input text is UTF-8
-    //      because native indexes are different from UTF-16 indexes.
-    //      Temporary hack: skip dictionary lookup for UTF-8 encoded text.
-    //      It wont give the right breaks, but it's better than a crash.
-    //
-    //      Check the type of the UText by checking its pFuncs field, which
-    //      is UText's function dispatch table.  It will be the same for all
-    //      UTF-8 UTexts and different for any other UText type.
-    //
-    //      We have no other type of UText available with non-UTF-16 native indexing.
-    //      This whole check will go away once the dictionary code is fixed.
-    static const void *utext_utf8Funcs;
-    if (utext_utf8Funcs == NULL) {
-        // Cache the UTF-8 UText function pointer value.
-        UErrorCode status = U_ZERO_ERROR;
-        UText tempUText = UTEXT_INITIALIZER; 
-        utext_openUTF8(&tempUText, NULL, 0, &status);
-        utext_utf8Funcs = tempUText.pFuncs;
-        utext_close(&tempUText);
-    }
-    if (fText->pFuncs == utext_utf8Funcs) {
-        return (reverse ? startPos : endPos);
-    }
-
     // Starting from the starting point, scan towards the proposed result,
     // looking for the first dictionary character (which may be the one
     // we're on, if we're starting in the middle of a range).
     utext_setNativeIndex(fText, reverse ? endPos : startPos);
     if (reverse) {
         UTEXT_PREVIOUS32(fText);
     }
     
     int32_t rangeStart = startPos;
     int32_t rangeEnd = endPos;
@@ skipping 81 matching lines @@
         }
         
         // Reload the loop variables for the next go-round
         c = utext_current32(fText);
         UTRIE_GET16(&fData->fTrie, c, category);
     }
     
     // If we found breaks, build a new break cache. The first and last entries must
     // be the original starting and ending position.
     if (foundBreakCount > 0) {
+        U_ASSERT(foundBreakCount == breaks.size());
         int32_t totalBreaks = foundBreakCount;
         if (startPos < breaks.elementAti(0)) {
             totalBreaks += 1;
         }
         if (endPos > breaks.peeki()) {
             totalBreaks += 1;
         }
         fCachedBreakPositions = (int32_t *)uprv_malloc(totalBreaks * sizeof(int32_t));
         if (fCachedBreakPositions != NULL) {
             int32_t out = 0;
@@ skipping 19 matching lines @@
         }
         // If the allocation failed, just fall through to the "no breaks found" case.
     }
 
     // If we get here, there were no language-based breaks. Set the text pointer
     // to the original proposed break.
     utext_setNativeIndex(fText, reverse ? startPos : endPos);
     return (reverse ? startPos : endPos);
 }
 
-// defined in ucln_cmn.h
-
 U_NAMESPACE_END
 
 
 static icu::UStack *gLanguageBreakFactories = NULL;
 static icu::UInitOnce gLanguageBreakFactoriesInitOnce = U_INITONCE_INITIALIZER;
 
 /**
  * Release all static memory held by breakiterator.  
  */
 U_CDECL_BEGIN
@@ skipping 125 matching lines @@
 }*/
 
 void RuleBasedBreakIterator::setBreakType(int32_t type) {
     fBreakType = type;
     reset();
 }
 
 U_NAMESPACE_END
 
 #endif /* #if !UCONFIG_NO_BREAK_ITERATION */