CJK segmentation patch for ICU 4.6...

icu46/patches/segmentation.patch

---- source/common/brkeng.cpp    2007-09-11 20:53:13.000000000 -0700
-+++ source/common/brkeng.cpp    2009-07-29 12:57:49.973382000 -0700
-@@ -24,6 +24,7 @@
- #include "umutex.h"
- #include "uresimp.h"
- #include "ubrkimpl.h"
-+#include <stdio.h>
- 
- U_NAMESPACE_BEGIN
- 
-@@ -226,6 +227,30 @@
+--- source/common/brkeng.cpp    2009-11-11 07:47:22.000000000 -0800
++++ source/common/brkeng.cpp    2011-01-21 14:12:45.479922000 -0800
+@@ -226,6 +226,30 @@
              case USCRIPT_THAI:
                  engine = new ThaiBreakEngine(dict, status);
                  break;
 +                
 +            case USCRIPT_HANGUL:
 +                engine = new CjkBreakEngine(dict, kKorean, status);
 +                break;
 +
 +            // use same BreakEngine and dictionary for both Chinese and Japanese
 +            case USCRIPT_HIRAGANA:
@@ skipping 10 matching lines @@
 +            {
 +                UBlockCode block = ublock_getCode(code);
 +                if (block == UBLOCK_HIRAGANA || block == UBLOCK_KATAKANA)
 +                   engine = new CjkBreakEngine(dict, kChineseJapanese, status);
 +                break;
 +            }
 +#endif
              default:
                  break;
              }
-@@ -281,6 +306,13 @@
+@@ -281,6 +305,13 @@
              dict = NULL;
          }
          return dict;
 +    } else if (dictfname != NULL){
 +        //create dummy dict if dictionary filename not valid
 +        UChar c = 0x0020;
 +        status = U_ZERO_ERROR;
 +        MutableTrieDictionary *mtd = new MutableTrieDictionary(c, status, TRUE);
 +        mtd->addWord(&c, 1, status, 1);
 +        return new CompactTrieDictionary(*mtd, status);  
      }
      return NULL;
  }
 --- source/common/dictbe.cpp    2008-06-13 12:21:12.000000000 -0700
-+++ source/common/dictbe.cpp    2009-11-11 12:58:40.199829000 -0800
-@@ -16,6 +16,11 @@
++++ source/common/dictbe.cpp    2011-01-21 14:12:45.468928000 -0800
+@@ -16,6 +16,9 @@
  #include "unicode/ubrk.h"
  #include "uvector.h"
  #include "triedict.h"
 +#include "uassert.h"
 +#include "unicode/normlzr.h"
 +#include "cmemory.h"
-+
-+#include <stdio.h>
  
  U_NAMESPACE_BEGIN
  
-@@ -422,6 +427,294 @@
+@@ -422,6 +425,294 @@
      return wordsFound;
  }
  
 +/*
 + ******************************************************************
 + * CjkBreakEngine
 + */
 +static const uint32_t kuint32max = 0xFFFFFFFF;
 +CjkBreakEngine::CjkBreakEngine(const TrieWordDictionary *adoptDictionary, LanguageType type, UErrorCode &status)
 +: DictionaryBreakEngine(1<<UBRK_WORD), fDictionary(adoptDictionary){
@@ skipping 275 matching lines @@
 +    }
 +
 +    utext_close(&normalizedText);
 +    return numBreaks;
 +}
 +
  U_NAMESPACE_END
  
  #endif /* #if !UCONFIG_NO_BREAK_ITERATION */
 --- source/common/dictbe.h      2006-09-29 17:37:45.000000000 -0700
-+++ source/common/dictbe.h      2009-07-27 13:01:17.704415000 -0700
++++ source/common/dictbe.h      2011-01-21 14:12:45.492920000 -0800
 @@ -1,8 +1,8 @@
  /**
 - *******************************************************************************
 - * Copyright (C) 2006, International Business Machines Corporation and others. *
 - * All Rights Reserved.                                                        *
 - *******************************************************************************
 + **********************************************************************************
-+ * Copyright (C) 2006,2007, International Business Machines Corporation and others.
++ * Copyright (C) 2006-2010, International Business Machines Corporation and others.
 + * All Rights Reserved.                                                       
 + **********************************************************************************
   */
  
  #ifndef DICTBE_H
-@@ -65,37 +65,37 @@
+@@ -65,31 +65,31 @@
     */
    virtual ~DictionaryBreakEngine();
  
 - /**
 -  * <p>Indicate whether this engine handles a particular character for
 -  * a particular kind of break.</p>
 -  *
 -  * @param c A character which begins a run that the engine might handle
 -  * @param breakType The type of text break which the caller wants to determine
 -  * @return TRUE if this engine handles the particular character and break
@@ skipping 32 matching lines @@
 +   * that starts from the first (or last) character in the range.
 +   * @param startPos The start of the run within the supplied text.
 +   * @param endPos The end of the run within the supplied text.
 +   * @param reverse Whether the caller is looking for breaks in a reverse
 +   * direction.
 +   * @param breakType The type of break desired, or -1.
 +   * @param foundBreaks An allocated C array of the breaks found, if any
 +   * @return The number of breaks found.
 +   */
    virtual int32_t findBreaks( UText *text,
--                              int32_t startPos,
--                              int32_t endPos,
--                              UBool reverse,
--                              int32_t breakType,
--                              UStack &foundBreaks ) const;
-+          int32_t startPos,
-+          int32_t endPos,
-+          UBool reverse,
-+          int32_t breakType,
-+          UStack &foundBreaks ) const;
- 
-  protected:
- 
+                               int32_t startPos,
+                               int32_t endPos,
 @@ -114,7 +114,7 @@
  //  virtual void setBreakTypes( uint32_t breakTypes );
  
   /**
 -  * <p>Divide up a range of known dictionary characters.</p>
 +  * <p>Divide up a range of known dictionary characters handled by this break engine.</p>
    *
    * @param text A UText representing the text
    * @param rangeStart The start of the range of dictionary characters
 @@ -171,7 +171,7 @@
  
   protected:
   /**
 -  * <p>Divide up a range of known dictionary characters.</p>
 +  * <p>Divide up a range of known dictionary characters handled by this break engine.</p>
    *
    * @param text A UText representing the text
    * @param rangeStart The start of the range of dictionary characters
-@@ -180,12 +180,72 @@
-   * @return The number of breaks found
-   */
-   virtual int32_t divideUpDictionaryRange( UText *text,
--                                           int32_t rangeStart,
--                                           int32_t rangeEnd,
--                                           UStack &foundBreaks ) const;
-+          int32_t rangeStart,
-+          int32_t rangeEnd,
-+          UStack &foundBreaks ) const;
+@@ -186,6 +186,66 @@
  
  };
  
 +/*******************************************************************
 + * CjkBreakEngine
 + */
 +
 +//indicates language/script that the CjkBreakEngine will handle
 +enum LanguageType {
 +    kKorean,
@@ skipping 46 matching lines @@
 +     */
 +  virtual int32_t divideUpDictionaryRange( UText *text,
 +          int32_t rangeStart,
 +          int32_t rangeEnd,
 +          UStack &foundBreaks ) const;
 +
 +};
  
  U_NAMESPACE_END
  
---- source/common/rbbi.cpp      2008-09-24 22:48:27.000000000 -0700
-+++ source/common/rbbi.cpp      2009-07-27 13:01:17.710416000 -0700
-@@ -29,6 +29,7 @@
- 
- #include "uassert.h"
- #include "uvector.h"
-+#include <stdio.h>
- 
- // if U_LOCAL_SERVICE_HOOK is defined, then localsvc.cpp is expected to be included.
- #if U_LOCAL_SERVICE_HOOK
-@@ -1552,10 +1553,14 @@
+--- source/common/rbbi.cpp      2010-07-22 17:15:37.000000000 -0700
++++ source/common/rbbi.cpp      2011-01-21 14:12:45.457938000 -0800
+@@ -1555,10 +1555,12 @@
                              int32_t endPos,
                              UBool reverse) {
      // Reset the old break cache first.
 -    uint32_t dictionaryCount = fDictionaryCharCount;
-+//    uint32_t dictionaryCount = fDictionaryCharCount;
      reset();
  
 -    if (dictionaryCount <= 1 || (endPos - startPos) <= 1) {
 +    // note: code segment below assumes that dictionary chars are in the 
 +    // startPos-endPos range
 +    // value returned should be next character in sequence
-+//  if (dictionaryCount <= 1 || (endPos - startPos) <= 1) {
 +    if ((endPos - startPos) <= 1) {
          return (reverse ? startPos : endPos);
      }
      
-@@ -1684,7 +1689,7 @@
+@@ -1711,7 +1713,7 @@
              // proposed break by one of the breaks we found. Use following() and
              // preceding() to do the work. They should never recurse in this case.
              if (reverse) {
 -                return preceding(endPos - 1);
 +                return preceding(endPos);
              }
              else {
                  return following(startPos);
 --- source/common/triedict.cpp  2008-02-13 01:35:50.000000000 -0800
-+++ source/common/triedict.cpp  2009-07-27 13:01:17.718409000 -0700
++++ source/common/triedict.cpp  2011-01-21 14:12:45.271006000 -0800
 @@ -20,6 +20,7 @@
  #include "uvector.h"
  #include "uvectr32.h"
  #include "uarrsort.h"
 +#include "hash.h"
  
  //#define DEBUG_TRIE_DICT 1
  
 @@ -27,6 +28,11 @@
  #include <sys/times.h>
@@ skipping 18 matching lines @@
 +
  // Node structure for the ternary, uncompressed trie
  struct TernaryNode : public UMemory {
      UChar       ch;         // UTF-16 code unit
 @@ -77,7 +88,8 @@
      delete high;
  }
  
 -MutableTrieDictionary::MutableTrieDictionary( UChar median, UErrorCode &status ) {
 +MutableTrieDictionary::MutableTrieDictionary( UChar median, UErrorCode &status,
-+        UBool containsValue /* = FALSE */  ) {
++                                              UBool containsValue /* = FALSE */  ) {
      // Start the trie off with something. Having the root node already present
      // cuts a special case out of the search/insertion functions.
      // Making it a median character cuts the worse case for searches from
 @@ -91,14 +103,19 @@
      if (U_SUCCESS(status) && fIter == NULL) {
          status = U_MEMORY_ALLOCATION_ERROR;
      }
 +
 +    fValued = containsValue;
  }
  
 -MutableTrieDictionary::MutableTrieDictionary( UErrorCode &status ) {
 +MutableTrieDictionary::MutableTrieDictionary( UErrorCode &status, 
-+        UBool containsValue /* = false */ ) {
++                                              UBool containsValue /* = false */ ) {
      fTrie = NULL;
      fIter = utext_openUChars(NULL, NULL, 0, &status);
      if (U_SUCCESS(status) && fIter == NULL) {
          status = U_MEMORY_ALLOCATION_ERROR;
      }
 +
 +    fValued = containsValue;
  }
  
  MutableTrieDictionary::~MutableTrieDictionary() {
-@@ -113,7 +130,8 @@
-                                    int &count,
-                                    int limit,
-                                    TernaryNode *&parent,
+@@ -108,12 +125,13 @@
+ 
+ int32_t
+ MutableTrieDictionary::search( UText *text,
+-                                   int32_t maxLength,
+-                                   int32_t *lengths,
+-                                   int &count,
+-                                   int limit,
+-                                   TernaryNode *&parent,
 -                                   UBool &pMatched ) const {
-+                                   UBool &pMatched,
-+                                   uint16_t *values /*=NULL*/) const {
++                               int32_t maxLength,
++                               int32_t *lengths,
++                               int &count,
++                               int limit,
++                               TernaryNode *&parent,
++                               UBool &pMatched,
++                               uint16_t *values /*=NULL*/) const {
      // TODO: current implementation works in UTF-16 space
      const TernaryNode *up = NULL;
      const TernaryNode *p = fTrie;
 @@ -121,6 +139,10 @@
      pMatched = TRUE;
      int i;
  
 +    if (!fValued) {
 +        values = NULL;
 +    }
@@ skipping 35 matching lines @@
      int count;
 @@ -177,7 +204,7 @@
      matched = search(fIter, length, NULL, count, 0, parent, pMatched);
      
      while (matched++ < length) {
 -        UChar32 uc = utext_next32(fIter);  // TODO:  supplemetary support?
 +        UChar32 uc = utext_next32(fIter);  // TODO:  supplementary support?
          U_ASSERT(uc != U_SENTINEL);
          TernaryNode *newNode = new TernaryNode(uc);
          if (newNode == NULL) {
-@@ -199,7 +226,11 @@
+@@ -199,30 +226,23 @@
          parent = newNode;
      }
  
 -    parent->flags |= kEndsWord;
+-}
+-
+-#if 0
+-void
+-MutableTrieDictionary::addWords( UEnumeration *words,
+-                                  UErrorCode &status ) {
+-    int32_t length;
+-    const UChar *word;
+-    while ((word = uenum_unext(words, &length, &status)) && U_SUCCESS(status)) {
+-        addWord(word, length, status);
 +    if(fValued && value > 0){
 +        parent->flags = value;
 +    } else {
 +        parent->flags |= kEndsWord;
-+    }
+     }
  }
+-#endif
  
- #if 0
-@@ -219,10 +250,11 @@
+ int32_t
+ MutableTrieDictionary::matches( UText *text,
                                  int32_t maxLength,
                                  int32_t *lengths,
                                  int &count,
 -                                int limit ) const {
 +                                int limit,
 +                                uint16_t *values /*=NULL*/) const {
      TernaryNode *parent;
      UBool pMatched;
 -    return search(text, maxLength, lengths, count, limit, parent, pMatched);
 +    return search(text, maxLength, lengths, count, limit, parent, pMatched, values);
  }
  
  // Implementation of iteration for MutableTrieDictionary
-@@ -277,7 +309,7 @@
+@@ -277,7 +297,7 @@
                      break;
                  }
              case kEqual:
 -                emit = (node->flags & kEndsWord) != 0;
 +                emit = node->flags > 0;
                  equal = (node->equal != NULL);
                  // If this node should be part of the next emitted string, append
                  // the UChar to the string, and make sure we pop it when we come
-@@ -299,7 +331,7 @@
+@@ -299,7 +319,7 @@
                  }
              case kGreaterThan:
                  // If this node's character is in the string, remove it.
 -                if (node->equal != NULL || (node->flags & kEndsWord)) {
 +                if (node->equal != NULL || node->flags > 0) {
                      unistr.truncate(unistr.length()-1);
                  }
                  if (node->high != NULL) {
-@@ -354,12 +386,74 @@
+@@ -354,12 +374,75 @@
   * CompactTrieDictionary
   */
  
-+//TODO if time permits: minimise size of trie with logprobs by storing values
++//TODO further optimization:
++// minimise size of trie with logprobs by storing values
 +// for terminal nodes directly in offsets[]
 +// --> calculating from next offset *might* be simpler, but would have to add
 +// one last offset for logprob of last node
 +// --> if calculate from current offset, need to factor in possible overflow
 +// as well.
 +// idea: store in offset, set first bit to indicate logprob storage-->won't
 +// have to access additional node
 +
 +// {'Dic', 1}, version 1: uses old header, no values
 +#define COMPACT_TRIE_MAGIC_1 0x44696301
@@ skipping 51 matching lines @@
 +                offsets = &(header->offsets[0]);
 +                address = (uint8_t *)header;
 +            }
 +        }
 +    }
 +
 +    ~CompactTrieInfo(){}
  };
  
  // Note that to avoid platform-specific alignment issues, all members of the node
-@@ -375,10 +469,14 @@
+@@ -375,10 +458,14 @@
  enum CompactTrieNodeFlags {
      kVerticalNode   = 0x1000,       // This is a vertical node
      kParentEndsWord = 0x2000,       // The node whose equal link points to this ends a word
 -    kReservedFlag1  = 0x4000,
 -    kReservedFlag2  = 0x8000,
 +    kExceedsCount   = 0x4000,       // new MSB for count >= 4096, originally kReservedFlag1
 +    kEqualOverflows = 0x8000,       // Links to nodeIDs > 2^16, orig. kReservedFlag2
      kCountMask      = 0x0FFF,       // The count portion of flagscount
 -    kFlagMask       = 0xF000        // The flags portion of flagscount
 +    kFlagMask       = 0xF000,       // The flags portion of flagscount
 +    kRootCountMask  = 0x7FFF        // The count portion of flagscount in the root node
 +
 +    //offset flags:
 +    //kOffsetContainsValue = 0x80000000       // Offset contains value for parent node
  };
  
  // The two node types are distinguished by the kVerticalNode flag.
-@@ -402,63 +500,177 @@
+@@ -402,63 +489,177 @@
      uint16_t        chars[1];       // Code units
  };
  
 -// {'Dic', 1}, version 1
 -#define COMPACT_TRIE_MAGIC_1 0x44696301
 -
  CompactTrieDictionary::CompactTrieDictionary(UDataMemory *dataObj,
                                                  UErrorCode &status )
  : fUData(dataObj)
  {
@@ skipping 54 matching lines @@
  uint32_t
  CompactTrieDictionary::dataSize() const {
 -    return fData->size;
 +    return fInfo->size;
  }
  
  const void *
  CompactTrieDictionary::data() const {
 -    return fData;
 +    return fInfo->address;
- }
- 
--// This function finds the address of a node for us, given its node ID
++}
++
 +//This function finds the address of a node for us, given its node ID
- static inline const CompactTrieNode *
--getCompactNode(const CompactTrieHeader *header, uint16_t node) {
--    return (const CompactTrieNode *)((const uint8_t *)header + header->offsets[node]);
++static inline const CompactTrieNode *
 +getCompactNode(const CompactTrieInfo *info, uint32_t node) {
 +    if(node < info->root-1) {
 +        return (const CompactTrieNode *)(&info->offsets[node]);
 +    } else {
 +        return (const CompactTrieNode *)(info->address + info->offsets[node]);
 +    }
-+}
-+
+ }
+ 
+-// This function finds the address of a node for us, given its node ID
 +//this version of getCompactNode is currently only used in compactMutableTrieDictionary()
-+static inline const CompactTrieNode *
+ static inline const CompactTrieNode *
+-getCompactNode(const CompactTrieHeader *header, uint16_t node) {
+-    return (const CompactTrieNode *)((const uint8_t *)header + header->offsets[node]);
 +getCompactNode(const CompactTrieHeader *header, uint32_t node) {
 +    if(node < header->root-1) {
 +        return (const CompactTrieNode *)(&header->offsets[node]);
 +    } else {
 +        return (const CompactTrieNode *)((const uint8_t *)header + header->offsets[node]);
 +    }
 +}
 +
 +
 +/**
@@ skipping 86 matching lines @@
 +        }
 +        else {
 +            low = middle+1;
 +        }
 +    }
 +
 +    return -1;
  }
  
  int32_t
-@@ -466,17 +678,38 @@
+@@ -466,17 +667,38 @@
                                  int32_t maxLength,
                                  int32_t *lengths,
                                  int &count,
 -                                int limit ) const {
 +                                int limit,
 +                                uint16_t *values /*= NULL*/) const {
 +    if (fInfo->magic == COMPACT_TRIE_MAGIC_2)
 +        values = NULL;
 +
      // TODO: current implementation works in UTF-16 space
@@ skipping 20 matching lines @@
 +
      while (node != NULL) {
          // Check if the node we just exited ends a word
          if (limit > 0 && (node->flagscount & kParentEndsWord)) {
 +            if(values != NULL){
 +                values[mycount] = getValue(node);
 +            }
              lengths[mycount++] = i;
              --limit;
          }
-@@ -487,7 +720,7 @@
+@@ -487,7 +709,7 @@
              break;
          }
  
 -        int nodeCount = (node->flagscount & kCountMask);
 +        int nodeCount = getCount(node);
          if (nodeCount == 0) {
              // Special terminal node; return now
              break;
-@@ -507,35 +740,27 @@
+@@ -507,35 +729,27 @@
              // To get here we must have come through the whole list successfully;
              // go on to the next node. Note that a word cannot end in the middle
              // of a vertical node.
 -            node = getCompactNode(fData, vnode->equal);
 +            node = getCompactNode(fInfo, calcEqualLink(vnode));
          }
          else {
              // Horizontal node; do binary search
              const CompactTrieHorizontalNode *hnode = (const CompactTrieHorizontalNode *)node;
 -            int low = 0;
@@ skipping 29 matching lines @@
 +            }else{
 +                node = NULL;    // If we don't find a match, we'll fall out of the loop              
              }
          }
      }
 -exit:
 +    exit:
      count = mycount;
      return i;
  }
-@@ -545,16 +770,16 @@
+@@ -545,16 +759,16 @@
  private:
      UVector32               fNodeStack;     // Stack of nodes to process
      UVector32               fIndexStack;    // Stack of where in node we are
 -    const CompactTrieHeader *fHeader;       // Trie data
 +    const CompactTrieInfo   *fInfo;         // Trie data
  
  public:
      static UClassID U_EXPORT2 getStaticClassID(void);
      virtual UClassID getDynamicClassID(void) const;
  public:
 -    CompactTrieEnumeration(const CompactTrieHeader *header, UErrorCode &status) 
 +    CompactTrieEnumeration(const CompactTrieInfo *info, UErrorCode &status) 
          : fNodeStack(status), fIndexStack(status) {
 -        fHeader = header;
 -        fNodeStack.push(header->root, status);
 +        fInfo = info;
 +        fNodeStack.push(info->root, status);
          fIndexStack.push(0, status);
          unistr.remove();
      }
-@@ -564,14 +789,14 @@
+@@ -564,14 +778,14 @@
      
      virtual StringEnumeration *clone() const {
          UErrorCode status = U_ZERO_ERROR;
 -        return new CompactTrieEnumeration(fHeader, status);
 +        return new CompactTrieEnumeration(fInfo, status);
      }
      
      virtual const UnicodeString * snext(UErrorCode &status);
  
      // Very expensive, but this should never be used.
      virtual int32_t count(UErrorCode &status) const {
 -        CompactTrieEnumeration counter(fHeader, status);
 +        CompactTrieEnumeration counter(fInfo, status);
          int32_t result = 0;
          while (counter.snext(status) != NULL && U_SUCCESS(status)) {
              ++result;
-@@ -582,7 +807,7 @@
+@@ -582,7 +796,7 @@
      virtual void reset(UErrorCode &status) {
          fNodeStack.removeAllElements();
          fIndexStack.removeAllElements();
 -        fNodeStack.push(fHeader->root, status);
 +        fNodeStack.push(fInfo->root, status);
          fIndexStack.push(0, status);
          unistr.remove();
      }
-@@ -595,26 +820,34 @@
+@@ -595,26 +809,34 @@
      if (fNodeStack.empty() || U_FAILURE(status)) {
          return NULL;
      }
 -    const CompactTrieNode *node = getCompactNode(fHeader, fNodeStack.peeki());
 +    const CompactTrieNode *node = getCompactNode(fInfo, fNodeStack.peeki());
      int where = fIndexStack.peeki();
      while (!fNodeStack.empty() && U_SUCCESS(status)) {
 -        int nodeCount = (node->flagscount & kCountMask);
 +        int nodeCount;
 +
@@ skipping 20 matching lines @@
              if (where == 0) {
                  // Going down
 -                unistr.append((const UChar *)vnode->chars, (int32_t) nodeCount);
 +                unistr.append((const UChar *)vnode->chars, nodeCount);
                  fIndexStack.setElementAt(1, fIndexStack.size()-1);
 -                node = getCompactNode(fHeader, fNodeStack.push(vnode->equal, status));
 +                node = getCompactNode(fInfo, fNodeStack.push(calcEqualLink(vnode), status));
                  where = fIndexStack.push(0, status);
                  goingDown = TRUE;
              }
-@@ -623,7 +856,7 @@
+@@ -623,7 +845,7 @@
                  unistr.truncate(unistr.length()-nodeCount);
                  fNodeStack.popi();
                  fIndexStack.popi();
 -                node = getCompactNode(fHeader, fNodeStack.peeki());
 +                node = getCompactNode(fInfo, fNodeStack.peeki());
                  where = fIndexStack.peeki();
              }
          }
-@@ -638,7 +871,7 @@
+@@ -638,7 +860,7 @@
                  // Push on next node
                  unistr.append((UChar)hnode->entries[where].ch);
                  fIndexStack.setElementAt(where+1, fIndexStack.size()-1);
 -                node = getCompactNode(fHeader, fNodeStack.push(hnode->entries[where].equal, status));
 +                node = getCompactNode(fInfo, fNodeStack.push(calcEqualLink(hnode, where, nodeCount), status));
                  where = fIndexStack.push(0, status);
                  goingDown = TRUE;
              }
-@@ -646,12 +879,14 @@
+@@ -646,12 +868,14 @@
                  // Going up
                  fNodeStack.popi();
                  fIndexStack.popi();
 -                node = getCompactNode(fHeader, fNodeStack.peeki());
 +                node = getCompactNode(fInfo, fNodeStack.peeki());
                  where = fIndexStack.peeki();
              }
          }
 +
          // Check if the parent of the node we've just gone down to ends a
          // word. If so, return it.
 +        // The root node should never end up here.
          if (goingDown && (node->flagscount & kParentEndsWord)) {
              return &unistr;
          }
-@@ -664,7 +899,7 @@
+@@ -664,7 +888,7 @@
      if (U_FAILURE(status)) {
          return NULL;
      }
 -    return new CompactTrieEnumeration(fData, status);
 +    return new CompactTrieEnumeration(fInfo, status);
  }
  
  //
-@@ -672,21 +907,36 @@
+@@ -672,21 +896,36 @@
  // and back again
  //
  
 -// Helper classes to construct the compact trie
 +enum CompactTrieNodeType {
 +    kHorizontalType = 0,
 +    kVerticalType = 1,
 +    kValueType = 2
 +};
 +
@@ skipping 22 matching lines @@
          fParentEndsWord = parentEndsWord;
          fHasDuplicate = FALSE;
 -        fVertical = vertical;
 +        fNodeType = nodeType;
 +        fEqualOverflows = FALSE;
          fNodeID = nodes.size();
 +        fValue = parentEndsWord? value : 0;
          nodes.push(this, status);
      }
      
-@@ -694,87 +944,225 @@
+@@ -694,87 +933,225 @@
      }
      
      virtual uint32_t size() {
 -        return sizeof(uint16_t);
 +        if(fValue > 0)
 +            return sizeof(uint16_t) * 2;
 +        else
 +            return sizeof(uint16_t);
      }
      
@@ skipping 221 matching lines @@
 +        // append 16 bits of to end for equal node if fEqualOverflows
 +        if (fEqualOverflows) {
 +            *((uint16_t *)(bytes+offset)) = (translate.elementAti(fEqual->fNodeID) >> 16);
 +            offset += sizeof(uint16_t);
 +        }
 +
 +        BuildCompactTrieNode::writeValue(bytes, offset);
      }
      
      void addChar(UChar ch) {
-@@ -784,60 +1172,85 @@
+@@ -784,60 +1161,85 @@
      void setLink(BuildCompactTrieNode *node) {
          fEqual = node;
      }
 +    
  };
  
  // Forward declaration
  static void walkHorizontal(const TernaryNode *node,
                              BuildCompactTrieHorizontalNode *building,
                              UStack &nodes,
@@ skipping 77 matching lines @@
 +                } else {
                      vResult->setLink((BuildCompactTrieNode *)nodes[1]);
                  }
              }
              else {
 -                vResult->setLink(compactOneNode(node, endsWord, nodes, status));
 +                vResult->setLink(compactOneNode(node, endsWord, nodes, status, values, value));
              }
              result = vResult;
          }
-@@ -849,19 +1262,28 @@
+@@ -849,19 +1251,28 @@
  // Uses recursion.
  
  static void walkHorizontal(const TernaryNode *node,
 -                            BuildCompactTrieHorizontalNode *building,
 -                            UStack &nodes,
 -                            UErrorCode &status) {
-+        BuildCompactTrieHorizontalNode *building,
-+        UStack &nodes,
-+        UErrorCode &status, Hashtable *values = NULL) {
++                           BuildCompactTrieHorizontalNode *building,
++                           UStack &nodes,
++                           UErrorCode &status, Hashtable *values = NULL) {
      while (U_SUCCESS(status) && node != NULL) {
          if (node->low != NULL) {
 -            walkHorizontal(node->low, building, nodes, status);
 +            walkHorizontal(node->low, building, nodes, status, values);
          }
          BuildCompactTrieNode *link = NULL;
          if (node->equal != NULL) {
 -            link = compactOneNode(node->equal, (node->flags & kEndsWord) != 0, nodes, status);
 +            link = compactOneNode(node->equal, node->flags > 0, nodes, status, values, node->flags);
          }
 -        else if (node->flags & kEndsWord) {
 -            link = (BuildCompactTrieNode *)nodes[1];
 +        else if (node->flags > 0) {
 +            if(values != NULL) {
 +                UnicodeString key(node->flags); //store value as a single-char UnicodeString
 +                link = (BuildCompactTrieValueNode *) values->get(key);
 +                if(link == NULL) {
 +                    link = new BuildCompactTrieValueNode(nodes, status, node->flags); //take out nodes?
 +                    values->put(key, link, status);
 +                }
 +            } else {
 +                link = (BuildCompactTrieNode *)nodes[1];
 +            }
          }
          if (U_SUCCESS(status) && link != NULL) {
              building->addNode(node->ch, link, status);
-@@ -881,13 +1303,15 @@
+@@ -881,13 +1292,15 @@
  _sortBuildNodes(const void * /*context*/, const void *voidl, const void *voidr) {
      BuildCompactTrieNode *left = *(BuildCompactTrieNode **)voidl;
      BuildCompactTrieNode *right = *(BuildCompactTrieNode **)voidr;
 +
      // Check for comparing a node to itself, to avoid spurious duplicates
      if (left == right) {
          return 0;
      }
 +
      // Most significant is type of node. Can never coalesce.
 -    if (left->fVertical != right->fVertical) {
 -        return left->fVertical - right->fVertical;
 +    if (left->fNodeType != right->fNodeType) {
 +        return left->fNodeType - right->fNodeType;
      }
      // Next, the "parent ends word" flag. If that differs, we cannot coalesce.
      if (left->fParentEndsWord != right->fParentEndsWord) {
-@@ -898,12 +1322,19 @@
+@@ -898,12 +1311,19 @@
      if (result != 0) {
          return result;
      }
 +
 +    // If the node value differs, we should not coalesce.
 +    // If values aren't stored, all fValues should be 0.
 +    if (left->fValue != right->fValue) {
 +        return left->fValue - right->fValue;
 +    }
 +
      // We know they're both the same node type, so branch for the two cases.
 -    if (left->fVertical) {
 +    if (left->fNodeType == kVerticalType) {
          result = ((BuildCompactTrieVerticalNode *)left)->fEqual->fNodeID
 -                            - ((BuildCompactTrieVerticalNode *)right)->fEqual->fNodeID;
 +        - ((BuildCompactTrieVerticalNode *)right)->fEqual->fNodeID;
      }
 -    else {
 +    else if(left->fChars.length() > 0 && right->fChars.length() > 0){
          // We need to compare the links vectors. They should be the
          // same size because the strings were equal.
          // We compare the node IDs instead of the pointers, to handle
-@@ -914,9 +1345,10 @@
+@@ -914,9 +1334,10 @@
          int32_t count = hleft->fLinks.size();
          for (int32_t i = 0; i < count && result == 0; ++i) {
              result = ((BuildCompactTrieNode *)(hleft->fLinks[i]))->fNodeID -
 -                     ((BuildCompactTrieNode *)(hright->fLinks[i]))->fNodeID;
 +            ((BuildCompactTrieNode *)(hright->fLinks[i]))->fNodeID;
          }
      }
 +
      // If they are equal to each other, mark them (speeds coalescing)
      if (result == 0) {
          left->fHasDuplicate = TRUE;
-@@ -1031,20 +1463,25 @@
+@@ -1031,20 +1452,25 @@
      // Add node 0, used as the NULL pointer/sentinel.
      nodes.addElement((int32_t)0, status);
  
 +    Hashtable *values = NULL;                           // Index of (unique) values
 +    if (dict.fValued) {
 +        values = new Hashtable(status);
 +    }
 +
      // Start by creating the special empty node we use to indicate that the parent
      // terminates a word. This must be node 1, because the builder assumes
 -    // that.
 +    // that. This node will never be used for tries storing numerical values.
      if (U_FAILURE(status)) {
          return NULL;
      }
 -    BuildCompactTrieNode *terminal = new BuildCompactTrieNode(TRUE, FALSE, nodes, status);
 +    BuildCompactTrieNode *terminal = new BuildCompactTrieNode(TRUE, kHorizontalType, nodes, status);
      if (terminal == NULL) {
          status = U_MEMORY_ALLOCATION_ERROR;
      }
  
      // This call does all the work of building the new trie structure. The root
 -    // will be node 2.
 -    BuildCompactTrieNode *root = compactOneNode(dict.fTrie, FALSE, nodes, status);
 +    // will have node ID 2 before writing to memory.
 +    BuildCompactTrieNode *root = compactOneNode(dict.fTrie, FALSE, nodes, status, values);
  #ifdef DEBUG_TRIE_DICT
      (void) ::times(&timing);
      fprintf(stderr, "Compact trie built, %d nodes, time user %f system %f\n",
-@@ -1077,21 +1514,37 @@
+@@ -1077,21 +1503,37 @@
          return NULL;
      }
  
 +    //map terminal value nodes
 +    int valueCount = 0;
 +    UVector valueNodes(status);
 +    if(values != NULL) {
 +        valueCount = values->count(); //number of unique terminal value nodes
 +    }
 +     
@@ skipping 23 matching lines @@
      }
 -    
 -    // Check for overflowing 16 bits worth of nodes.
 -    if (nodeCount > 0x10000) {
 +
 +    // Check for overflowing 20 bits worth of nodes.
 +    if (nodeCount > 0x100000) {
          status = U_ILLEGAL_ARGUMENT_ERROR;
          return NULL;
      }
-@@ -1111,9 +1564,14 @@
+@@ -1111,9 +1553,14 @@
          status = U_MEMORY_ALLOCATION_ERROR;
          return NULL;
      }
 -
 +    
      CompactTrieHeader *header = (CompactTrieHeader *)bytes;
 -    header->size = totalSize;
 +    //header->size = totalSize;
 +    if(dict.fValued){
 +        header->magic = COMPACT_TRIE_MAGIC_3;
 +    } else {
 +        header->magic = COMPACT_TRIE_MAGIC_2;
 +    }
      header->nodeCount = nodeCount;
      header->offsets[0] = 0;                     // Sentinel
      header->root = translate.elementAti(root->fNodeID);
-@@ -1123,23 +1581,40 @@
+@@ -1123,23 +1570,40 @@
      }
  #endif
      uint32_t offset = offsetof(CompactTrieHeader,offsets)+(nodeCount*sizeof(uint32_t));
 -    nodeCount = 1;
 +    nodeCount = valueCount + 1;
 +
 +    // Write terminal value nodes to memory
 +    for (i=0; i < valueNodes.size(); i++) {
 +        //header->offsets[i + 1] = offset;
 +        uint32_t tmpOffset = 0;
@@ skipping 23 matching lines @@
      fprintf(stderr, "Trie built, time user %f system %f\n",
          (double)(timing.tms_utime-previous.tms_utime)/CLK_TCK,
          (double)(timing.tms_stime-previous.tms_stime)/CLK_TCK);
      previous = timing;
      fprintf(stderr, "Final offset is %d\n", offset);
 -    
 +
      // Collect statistics on node types and sizes
      int hCount = 0;
      int vCount = 0;
-@@ -1148,68 +1623,85 @@
+@@ -1148,68 +1612,85 @@
      size_t hItemCount = 0;
      size_t vItemCount = 0;
      uint32_t previousOff = offset;
 -    for (uint16_t nodeIdx = nodeCount-1; nodeIdx >= 2; --nodeIdx) {
 +    uint32_t numOverflow = 0;
 +    uint32_t valueSpace = 0;
 +    for (uint32_t nodeIdx = nodeCount-1; nodeIdx >= 2; --nodeIdx) {
          const CompactTrieNode *node = getCompactNode(header, nodeIdx);
 -        if (node->flagscount & kVerticalNode) {
 +        int itemCount;
@@ skipping 85 matching lines @@
  
  // Convert one compact trie node into a ternary subtrie
  static TernaryNode *
 -unpackOneNode( const CompactTrieHeader *header, const CompactTrieNode *node, UErrorCode &status ) {
 -    int nodeCount = (node->flagscount & kCountMask);
 +unpackOneNode( const CompactTrieInfo *info, const CompactTrieNode *node, UErrorCode &status ) {
 +    int nodeCount = getCount(node);
      if (nodeCount == 0 || U_FAILURE(status)) {
          // Failure, or terminal node
          return NULL;
-@@ -1234,29 +1726,41 @@
+@@ -1234,29 +1715,41 @@
              previous = latest;
          }
          if (latest != NULL) {
 -            const CompactTrieNode *equal = getCompactNode(header, vnode->equal);
 +            const CompactTrieNode *equal = getCompactNode(info, calcEqualLink(vnode));
              if (equal->flagscount & kParentEndsWord) {
 -                latest->flags |= kEndsWord;
 +                if(info->magic == COMPACT_TRIE_MAGIC_3){
 +                    latest->flags = getValue(equal);
 +                } else {
@@ skipping 27 matching lines @@
 +    // because only kEqualOverflows flag should be checked in root's flagscount
 +    const CompactTrieHorizontalNode *hnode = (const CompactTrieHorizontalNode *) 
 +    getCompactNode(fInfo, fInfo->root);
 +    uint16_t nodeCount = hnode->flagscount & kRootCountMask;
 +    TernaryNode *root = unpackHorizontalArray(fInfo, hnode, 0, nodeCount-1, 
 +            nodeCount, status);
 +
      if (U_FAILURE(status)) {
          delete root;    // Clean up
          delete result;
-@@ -1270,8 +1774,8 @@
+@@ -1270,8 +1763,8 @@
  
  U_CAPI int32_t U_EXPORT2
  triedict_swap(const UDataSwapper *ds, const void *inData, int32_t length, void *outData,
 -           UErrorCode *status) {
 -
 +        UErrorCode *status) {
 +    
      if (status == NULL || U_FAILURE(*status)) {
          return 0;
      }
-@@ -1286,14 +1790,14 @@
+@@ -1286,14 +1779,14 @@
      //
      const UDataInfo *pInfo = (const UDataInfo *)((const uint8_t *)inData+4);
      if(!(  pInfo->dataFormat[0]==0x54 &&   /* dataFormat="TrDc" */
 -           pInfo->dataFormat[1]==0x72 &&
 -           pInfo->dataFormat[2]==0x44 &&
 -           pInfo->dataFormat[3]==0x63 &&
 -           pInfo->formatVersion[0]==1  )) {
 +            pInfo->dataFormat[1]==0x72 &&
 +            pInfo->dataFormat[2]==0x44 &&
 +            pInfo->dataFormat[3]==0x63 &&
 +            pInfo->formatVersion[0]==1  )) {
          udata_printError(ds, "triedict_swap(): data format %02x.%02x.%02x.%02x (format version %02x) is not recognized\n",
 -                         pInfo->dataFormat[0], pInfo->dataFormat[1],
 -                         pInfo->dataFormat[2], pInfo->dataFormat[3],
 -                         pInfo->formatVersion[0]);
 +                pInfo->dataFormat[0], pInfo->dataFormat[1],
 +                pInfo->dataFormat[2], pInfo->dataFormat[3],
 +                pInfo->formatVersion[0]);
          *status=U_UNSUPPORTED_ERROR;
          return 0;
      }
-@@ -1311,8 +1815,10 @@
+@@ -1311,8 +1804,10 @@
      //
      const uint8_t  *inBytes =(const uint8_t *)inData+headerSize;
      const CompactTrieHeader *header = (const CompactTrieHeader *)inBytes;
 -    if (ds->readUInt32(header->magic) != COMPACT_TRIE_MAGIC_1
 -            || ds->readUInt32(header->size) < sizeof(CompactTrieHeader))
 +    uint32_t magic = ds->readUInt32(header->magic);
 +    if (magic != COMPACT_TRIE_MAGIC_1 && magic != COMPACT_TRIE_MAGIC_2 && magic != COMPACT_TRIE_MAGIC_3
 +            || magic == COMPACT_TRIE_MAGIC_1 && ds->readUInt32(header->size) < sizeof(CompactTrieHeaderV1)
 +            || magic != COMPACT_TRIE_MAGIC_1 && ds->readUInt32(header->size) < sizeof(CompactTrieHeader))
      {
          udata_printError(ds, "triedict_swap(): CompactTrieHeader is invalid.\n");
          *status=U_UNSUPPORTED_ERROR;
-@@ -1333,10 +1839,10 @@
+@@ -1333,10 +1828,10 @@
      //
      if (length < sizeWithUData) {
          udata_printError(ds, "triedict_swap(): too few bytes (%d after ICU Data header) for trie data.\n",
 -                            totalSize);
 +                totalSize);
          *status=U_INDEX_OUTOFBOUNDS_ERROR;
          return 0;
 -        }
 +    }
  
      //
      // Swap the Data.  Do the data itself first, then the CompactTrieHeader, because
-@@ -1355,20 +1861,38 @@
+@@ -1355,20 +1850,38 @@
      }
  
      // We need to loop through all the nodes in the offset table, and swap each one.
 -    uint16_t nodeCount = ds->readUInt16(header->nodeCount);
 +    uint32_t nodeCount, rootId;
 +    if(header->magic == COMPACT_TRIE_MAGIC_1) {
 +        nodeCount = ds->readUInt16(((CompactTrieHeaderV1 *)header)->nodeCount);
 +        rootId = ds->readUInt16(((CompactTrieHeaderV1 *)header)->root);
 +    } else {
 +        nodeCount = ds->readUInt32(header->nodeCount);
@@ skipping 24 matching lines @@
 +                    overflow += 1;
 +                }
                  ds->swapArray16(ds, inBytes+nodeOff+offsetof(CompactTrieVerticalNode,chars),
 -                                    itemCount*sizeof(uint16_t),
 -                                    outBytes+nodeOff+offsetof(CompactTrieVerticalNode,chars), status);
 +                        (itemCount + overflow)*sizeof(uint16_t),
 +                        outBytes+nodeOff+offsetof(CompactTrieVerticalNode,chars), status);
                  uint16_t equal = ds->readUInt16(inBytes+nodeOff+offsetof(CompactTrieVerticalNode,equal);
                  ds->writeUInt16(outBytes+nodeOff+offsetof(CompactTrieVerticalNode,equal));
              }
-@@ -1381,26 +1905,62 @@
+@@ -1381,26 +1894,62 @@
                      word = ds->readUInt16(inHNode->entries[j].equal);
                      ds->writeUInt16(&outHNode->entries[j].equal, word);
                  }
 +
 +                // swap overflow/value information
 +                if(flagscount & kEqualOverflows){
 +                    overflow += (itemCount + 3) / 4;
 +                }
 +
 +                if (header->magic == COMPACT_TRIE_MAGIC_3 && i != rootId && flagscount & kEndsParentWord) {
@@ skipping 55 matching lines @@
 +    ds->swapArray16(ds, inBytes+nodesOff, totalSize-nodesOff, outBytes+nodesOff, status);
 +
 +    //swap offsets
 +    ds->swapArray32(ds, inBytes+offsetPos,
 +            sizeof(uint32_t)*(uint32_t)nodeCount,
 +            outBytes+offsetPos, status);
  
      return sizeWithUData;
  }
 --- source/common/triedict.h    2006-06-06 15:38:49.000000000 -0700
-+++ source/common/triedict.h    2009-07-27 13:01:17.723390000 -0700
++++ source/common/triedict.h    2011-01-21 14:12:45.496927000 -0800
 @@ -47,7 +47,6 @@
  U_NAMESPACE_BEGIN
  
  class StringEnumeration;
 -struct CompactTrieHeader;
  
  /*******************************************************************
   * TrieWordDictionary
 @@ -72,23 +71,29 @@
     */
@@ skipping 218 matching lines @@
    *
    * @return The data for the compact dictionary, suitable for passing to the
    * constructor.
 @@ -342,5 +386,5 @@
  
  U_NAMESPACE_END
  
 -    /* TRIEDICT_H */
 +/* TRIEDICT_H */
  #endif
---- source/data/brkitr/brkfiles.mk      2009-04-21 15:42:37.000000000 -0700
-+++ source/data/brkitr/brkfiles.mk      2009-07-27 13:01:17.730379000 -0700
-@@ -34,13 +34,12 @@
+--- source/data/Makefile.in     2010-10-29 13:21:33.000000000 -0700
++++ source/data/Makefile.in     2011-01-26 16:24:24.856798000 -0800
+@@ -509,8 +520,9 @@
+ ####################################################    CTD
+ # CTD FILES
  
+-$(BRKBLDDIR)/%.ctd: $(BRKSRCDIR)/%.txt $(TOOLBINDIR)/genctd$(TOOLEXEEXT) $(DAT_FILES)
+-       $(INVOKE) $(TOOLBINDIR)/genctd -c -i $(BUILDDIR) -o $@ $<
++# .ctd file now generated regardless of whether dictionary file exists
++$(BRKBLDDIR)/%.ctd: $(TOOLBINDIR)/genctd$(TOOLEXEEXT) $(DAT_FILES)
++       $(INVOKE) $(TOOLBINDIR)/genctd -c -i $(BUILDDIR) -o $@ $(BRKSRCDIR)/$(*F).txt
  
- # List of compact trie dictionary files (ctd).
--BRK_CTD_SOURCE = thaidict.txt 
-+BRK_CTD_SOURCE = thaidict.txt cjdict.txt
- 
- 
- # List of break iterator files (brk).
--BRK_SOURCE = word_POSIX.txt word_ja.txt sent_el.txt char_th.txt char.txt word.txt line.txt sent.txt title.txt 
-+BRK_SOURCE = word_POSIX.txt sent_el.txt char_th.txt char.txt word.txt line.txt sent.txt title.txt 
- 
- 
- # Ordinary resources
--BRK_RES_SOURCE = el.txt en.txt en_US.txt en_US_POSIX.txt ja.txt th.txt
--
-+BRK_RES_SOURCE = el.txt en.txt en_US.txt en_US_POSIX.txt th.txt
---- source/data/brkitr/root.txt 2009-06-24 14:06:38.000000000 -0700
-+++ source/data/brkitr/root.txt 2009-07-27 13:01:17.733382000 -0700
+ ####################################################    CFU
+ # CFU FILES
+--- source/data/brkitr/root.txt 2010-07-28 17:18:28.000000000 -0700
++++ source/data/brkitr/root.txt 2011-01-21 14:12:45.653922000 -0800
 @@ -17,5 +17,8 @@
      }
      dictionaries{
          Thai:process(dependency){"thaidict.ctd"}
 +        Hani:process(dependency){"cjdict.ctd"}
 +        Hira:process(dependency){"cjdict.ctd"}
 +        Kata:process(dependency){"cjdict.ctd"}
      }
  }
---- source/data/brkitr/word.txt 2009-06-24 14:06:38.000000000 -0700
-+++ source/data/brkitr/word.txt 2010-08-27 16:24:25.969372000 -0700
-@@ -29,29 +29,49 @@
- $Newline      = [\p{Word_Break = Newline}];
- $Extend       = [\p{Word_Break = Extend}];
- $Format       = [\p{Word_Break = Format}];
-+$Hiragana     = [:Hiragana:];
- $Katakana     = [\p{Word_Break = Katakana}];
-+$Han          = [:Han:];
- $ALetter      = [\p{Word_Break = ALetter}];
--$MidNumLet    = [\p{Word_Break = MidNumLet}];
-+# Remove two full stop characters from $MidNumLet and add them to $MidNum
-+# to break a hostname into its components at the cost of breaking
-+# 'e.g.' and 'i.e.' as well.
-+# $MidNumLet is used in rules 6/7 (rules of our interest) and rules 11/12.
-+# Because it's OR'd with $MidNum in rules 11/12, rules 11/12 are not affected
-+# while rules 6/7 are reverted to the old behavior we want.
-+$MidNumLet    = [[\p{Word_Break = MidNumLet}] - [\u002E \uFF0E]];
- $MidLetter    = [\p{Word_Break = MidLetter}];
--$MidNum       = [\p{Word_Break = MidNum}];
--$Numeric      = [\p{Word_Break = Numeric}];
-+$MidNum       = [\p{Word_Break = MidNum}[\u002E \uFF0E]];
-+$Numeric      = [\p{Word_Break = Numeric}[\uff10-\uff19]]; #includes fullwidth digits
- $ExtendNumLet = [\p{Word_Break = ExtendNumLet}];
- 
-+# Extra sets not to break 'HebrewLetter U+0022 HebrewLetter'.
-+$HebrewLet    = [\p{Word_Break = ALetter} & \p{Script = Hebrew} - [\u05F3]];
-+# U+05F3 is ALetter and U+05F4 is MidLetter so that they're covered by
-+# the current rule 6/7.
-+$HebrewMidLet = [\u0022];
- 
- #   Dictionary character set, for triggering language-based break engines. Currently
--#   limited to LineBreak=Complex_Context. Note that this set only works in Unicode
--#   5.0 or later as the definition of Complex_Context was corrected to include all
-+#   limited to LineBreak=Complex_Context and CJK. Note that this set only works
-+#   in Unicode 5.0 or later as the definition of Complex_Context was corrected to include all
- #   characters requiring dictionary break.
- 
--$dictionary   = [:LineBreak = Complex_Context:];
- $Control        = [\p{Grapheme_Cluster_Break = Control}]; 
--$ALetterPlus  = [$ALetter [$dictionary-$Extend-$Control]];   # Note:  default ALetter does not
--                                                             #  include the dictionary characters.
-+$HangulSyllable = [\uac00-\ud7a3];
-+$ComplexContext = [:LineBreak = Complex_Context:];
-+$KanaKanji      = [$Han $Hiragana $Katakana];
-+$dictionaryCJK  = [$KanaKanji $HangulSyllable];
-+$dictionary     = [$ComplexContext $dictionaryCJK];
-+
-+# leave CJK scripts out of ALetterPlus
-+$ALetterPlus  = [$ALetter-$dictionaryCJK [$ComplexContext-$Extend-$Control]];
-+
- 
- #
- #  Rules 4    Ignore Format and Extend characters, 
- #             except when they appear at the beginning of a region of text.
- #
-+# TODO: check if handling of katakana in dictionary makes rules incorrect/void.
- $KatakanaEx     = $Katakana     ($Extend |  $Format)*;
- $ALetterEx      = $ALetterPlus  ($Extend |  $Format)*;
- $MidNumLetEx    = $MidNumLet    ($Extend |  $Format)*;
-@@ -59,8 +79,8 @@
- $MidNumEx       = $MidNum       ($Extend |  $Format)*;
- $NumericEx      = $Numeric      ($Extend |  $Format)*;
- $ExtendNumLetEx = $ExtendNumLet ($Extend |  $Format)*;
-+$HebrewLetEx    = $HebrewLet    ($Extend |  $Format)*;
- 
--$Hiragana       = [\p{script=Hiragana}];
- $Ideographic    = [\p{Ideographic}];
- $HiraganaEx     = $Hiragana     ($Extend |  $Format)*;
- $IdeographicEx  = $Ideographic  ($Extend |  $Format)*;
-@@ -79,12 +99,14 @@
- #          begins with a group of Format chars, or with a "word" consisting of a single
- #          char that is not in any of the listed word break categories followed by
- #          format char(s).
--[^$CR $LF $Newline]? ($Extend |  $Format)+;
-+ #          format char(s), or is not a CJK dictionary character.
-+[^$CR $LF $Newline $dictionaryCJK]? ($Extend |  $Format)+;
- 
- $NumericEx {100};
- $ALetterEx {200};
--$KatakanaEx {300};       # note:  these status values override those from rule 5
--$HiraganaEx {300};       #        by virtual of being numerically larger.
-+$HangulSyllable {200};
-+$KatakanaEx {400}; #originally 300
-+$HiraganaEx {400}; #originally 300
- $IdeographicEx {400};    #
- 
- #
-@@ -96,6 +118,9 @@
- # rule 6 and 7
- $ALetterEx ($MidLetterEx | $MidNumLetEx) $ALetterEx {200};
- 
-+# Chrome addition
-+$HebrewLetEx $HebrewMidLet $HebrewLetEx {200};
-+
- # rule 8
- 
- $NumericEx $NumericEx {100};
-@@ -114,19 +139,25 @@
- 
- # rule 13
- 
--$KatakanaEx  $KatakanaEx {300};
-+# To be consistent with '$KanaKanji $KanaKanji', changed 
-+# from 300 to 400.
-+# See also TestRuleStatus in intltest/rbbiapts.cpp
-+$KatakanaEx  $KatakanaEx {400};
- 
- # rule 13a/b
- 
- $ALetterEx      $ExtendNumLetEx {200};    #  (13a)
- $NumericEx      $ExtendNumLetEx {100};    #  (13a)
--$KatakanaEx     $ExtendNumLetEx {300};    #  (13a)
-+$KatakanaEx     $ExtendNumLetEx {400};    #  (13a)
- $ExtendNumLetEx $ExtendNumLetEx {200};    #  (13a)
- 
- $ExtendNumLetEx $ALetterEx  {200};    #  (13b)
- $ExtendNumLetEx $NumericEx  {100};    #  (13b)
--$ExtendNumLetEx $KatakanaEx {300};    #  (13b)
-- 
-+$ExtendNumLetEx $KatakanaEx {400};    #  (13b)
-+
-+# special handling for CJK characters: chain for later dictionary segmentation
-+$HangulSyllable $HangulSyllable {200};
-+$KanaKanji $KanaKanji {400}; #different rule status if both kanji and kana found
- 
- 
- ## -------------------------------------------------
-@@ -139,13 +170,15 @@
- $BackMidNumEx      = ($Format | $Extend)* $MidNum;
- $BackMidLetterEx   = ($Format | $Extend)* $MidLetter;
- $BackKatakanaEx    = ($Format | $Extend)* $Katakana;
-+$BackHiraganaEx    = ($Extend | $Format)* $Hiragana;
- $BackExtendNumLetEx= ($Format | $Extend)* $ExtendNumLet;
-+$BackHebrewLetEx   = ($Format | $Extend)* $HebrewLet;
- 
- # rule 3
- $LF $CR;
- 
- # rule 4
--($Format | $Extend)*  [^$CR $LF $Newline]?;
-+($Format | $Extend)*  [^$CR $LF $Newline $dictionaryCJK]?;
- 
- # rule 5
- 
-@@ -155,6 +188,8 @@
- 
- $BackALetterEx ($BackMidLetterEx | $BackMidNumLetEx) $BackALetterEx;
- 
-+# Chrome addition
-+$BackHebrewLetEx $HebrewMidLet $BackHebrewLetEx;
- 
- # rule 8
- 
-@@ -181,6 +216,10 @@
- $BackExtendNumLetEx ($BackALetterEx | $BackNumericEx | $BackKatakanaEx | $BackExtendNumLetEx);
- ($BackALetterEx | $BackNumericEx | $BackKatakanaEx) $BackExtendNumLetEx; 
- 
-+# special handling for CJK characters: chain for later dictionary segmentation
-+$HangulSyllable $HangulSyllable;
-+$KanaKanji $KanaKanji; #different rule status if both kanji and kana found
-+
- ## -------------------------------------------------
- 
- !!safe_reverse;
---- source/data/xml/brkitr/root.xml     2007-08-28 23:10:43.000000000 -0700
-+++ source/data/xml/brkitr/root.xml     2009-07-27 13:01:17.746367000 -0700
+--- source/data/xml/brkitr/root.xml     2010-03-01 15:13:18.000000000 -0800
++++ source/data/xml/brkitr/root.xml     2011-01-21 14:12:45.735922000 -0800
 @@ -25,6 +25,9 @@
              </icu:boundaries>
              <icu:dictionaries>
                  <icu:dictionary type="Thai" icu:dependency="thaidict.ctd"/>
 +                <icu:dictionary type="Hani" icu:dependency="cjdict.ctd"/>
 +                <icu:dictionary type="Hira" icu:dependency="cjdict.ctd"/>
 +                <icu:dictionary type="Kata" icu:dependency="cjdict.ctd"/>
              </icu:dictionaries>
          </icu:breakIteratorData>
      </special>
---- source/test/cintltst/creststn.c     2009-06-26 09:49:55.000000000 -0700
-+++ source/test/cintltst/creststn.c     2009-07-29 12:46:05.997405000 -0700
-@@ -2181,21 +2181,21 @@
+--- source/test/cintltst/creststn.c     2010-10-28 10:44:02.000000000 -0700
++++ source/test/cintltst/creststn.c     2011-01-21 14:12:44.995020000 -0800
+@@ -2188,21 +2188,21 @@
        
  
        {
 -            UResourceBundle* ja = ures_open(U_ICUDATA_BRKITR,"ja", &status);
 +            UResourceBundle* th = ures_open(U_ICUDATA_BRKITR,"th", &status);
              const UChar *got = NULL, *exp=NULL;
              int32_t gotLen = 0, expLen=0;
 -            ja = ures_getByKey(ja, "boundaries", ja, &status);
 -            exp = tres_getString(ja, -1, "word", &expLen, &status);
 +            th = ures_getByKey(th, "boundaries", th, &status);
 +            exp = tres_getString(th, -1, "grapheme", &expLen, &status);
                
              tb = ures_getByKey(aliasB, "boundaries", tb, &status);
 -            got = tres_getString(tb, -1, "word", &gotLen, &status);
 +            got = tres_getString(tb, -1, "grapheme", &gotLen, &status);
                  
              if(U_FAILURE(status)) {
                  log_err("%s trying to read str boundaries\n", u_errorName(status));
              } else if(gotLen != expLen || u_strncmp(exp, got, gotLen) != 0) {
                  log_err("Referencing alias didn't get the right data\n");
              }
 -            ures_close(ja);
 +            ures_close(th);
              status = U_ZERO_ERROR;
        }
        /* simple alias */
-@@ -3024,4 +3024,3 @@
-   }
- 
- }
--
---- source/test/intltest/rbbiapts.cpp   2009-06-26 09:49:55.000000000 -0700
-+++ source/test/intltest/rbbiapts.cpp   2009-07-28 13:56:30.208042000 -0700
+--- source/test/intltest/rbbiapts.cpp   2010-07-12 11:03:29.000000000 -0700
++++ source/test/intltest/rbbiapts.cpp   2011-01-21 14:12:45.033014000 -0800
 @@ -156,9 +156,13 @@
      if(*a!=*b){
          errln("Failed: boilerplate method operator!= does not return correct results");
      }
 -    BreakIterator* c = BreakIterator::createWordInstance(Locale("ja"),status);
 -    if(a && c){
 -        if(*c==*a){
 +    // Japanese word break iteratos is identical to root with
 +    // a dictionary-based break iterator, but Thai character break iterator
 +    // is still different from Root. 
 +    BreakIterator* c = BreakIterator::createCharacterInstance(Locale("ja"),status);
 +    BreakIterator* d = BreakIterator::createCharacterInstance(Locale("th"),status);
 +    if(c && d){
 +        if(*c==*d){
              errln("Failed: boilerplate method opertator== does not return correct results");
          }
      }else{
 @@ -167,6 +171,7 @@
      delete a;
      delete b;
      delete c;
 +    delete d;
  }
  
  void RBBIAPITest::TestgetRules()
-@@ -643,21 +648,21 @@
+@@ -635,21 +640,21 @@
  //
  void RBBIAPITest::TestRuleStatus() {
       UChar str[30];
 -     u_unescape("plain word 123.45 \\u9160\\u9161 \\u30a1\\u30a2 \\u3041\\u3094",
 -              // 012345678901234567  8      9    0  1      2    3  4      5    6
 -              //                    Ideographic    Katakana       Hiragana
 +     //no longer test Han or hiragana breaking here: ruleStatusVec would return nothing
 +     // changed UBRK_WORD_KANA to UBRK_WORD_IDEO
 +     u_unescape("plain word 123.45 \\u30a1\\u30a2 ",
 +              // 012345678901234567  8      9    0      
 +              //                     Katakana      
                  str, 30);
       UnicodeString testString1(str);
 -     int32_t bounds1[] = {0, 5, 6, 10, 11, 17, 18, 19, 20, 21, 23, 24, 25, 26};
 +     int32_t bounds1[] = {0, 5, 6, 10, 11, 17, 18, 20, 21};
       int32_t tag_lo[]  = {UBRK_WORD_NONE,     UBRK_WORD_LETTER, UBRK_WORD_NONE,    UBRK_WORD_LETTER,
                            UBRK_WORD_NONE,     UBRK_WORD_NUMBER, UBRK_WORD_NONE,
 -                          UBRK_WORD_IDEO,     UBRK_WORD_IDEO,   UBRK_WORD_NONE,
 -                          UBRK_WORD_KANA,     UBRK_WORD_NONE,   UBRK_WORD_KANA,    UBRK_WORD_KANA};
 +                          UBRK_WORD_IDEO,     UBRK_WORD_NONE};
  
       int32_t tag_hi[]  = {UBRK_WORD_NONE_LIMIT, UBRK_WORD_LETTER_LIMIT, UBRK_WORD_NONE_LIMIT, UBRK_WORD_LETTER_LIMIT,
                            UBRK_WORD_NONE_LIMIT, UBRK_WORD_NUMBER_LIMIT, UBRK_WORD_NONE_LIMIT,
 -                          UBRK_WORD_IDEO_LIMIT, UBRK_WORD_IDEO_LIMIT,   UBRK_WORD_NONE_LIMIT,
 -                          UBRK_WORD_KANA_LIMIT, UBRK_WORD_NONE_LIMIT,   UBRK_WORD_KANA_LIMIT, UBRK_WORD_KANA_LIMIT};
 +                          UBRK_WORD_IDEO_LIMIT, UBRK_WORD_NONE_LIMIT};
  
       UErrorCode status=U_ZERO_ERROR;
  
-@@ -896,9 +901,11 @@
+@@ -888,9 +893,11 @@
  
      URegistryKey key = BreakIterator::registerInstance(ja_word, "xx", UBRK_WORD, status);
      {
 +#if 0 // With a dictionary based word breaking, ja_word is identical to root.
          if (ja_word && *ja_word == *root_word) {
              errln("japan not different from root");
          }
 +#endif
      }
  
      {
---- source/test/intltest/rbbitst.cpp    2009-06-26 09:49:55.000000000 -0700
-+++ source/test/intltest/rbbitst.cpp    2009-07-28 15:35:18.933226000 -0700
-@@ -33,6 +33,8 @@
+--- source/test/intltest/rbbitst.cpp    2010-10-08 18:23:28.000000000 -0700
++++ source/test/intltest/rbbitst.cpp    2011-01-21 14:12:45.180030000 -0800
+@@ -35,6 +35,8 @@
  #include <string.h>
  #include <stdio.h>
  #include <stdlib.h>
 +#include "unicode/numfmt.h"
 +#include "unicode/uscript.h"
  
  #define TEST_ASSERT(x) {if (!(x)) { \
      errln("Failure in file %s, line %d", __FILE__, __LINE__);}}
-@@ -108,6 +110,8 @@
+@@ -138,11 +140,13 @@
              if (exec) TestThaiBreaks();                        break;
          case 23: name = "TestTailoredBreaks";
              if (exec) TestTailoredBreaks();                    break;
 +        case 24: name = "TestTrieDictWithValue";
 +            if(exec) TestTrieDictWithValue();                  break;
- 
-         default: name = ""; break; //needed to end loop
-     }
-@@ -570,6 +574,8 @@
+ #else
+-        case 21: case 22: case 23: name = "skip";
++        case 21: case 22: case 23: case 24: name = "skip";
+             break;
+ #endif
+-        case 24: name = "TestDictRules";
++        case 25: name = "TestDictRules";
+             if (exec) TestDictRules();                         break;
+         case 25: name = "TestBug5532";
+             if (exec) TestBug5532();                           break;
+@@ -607,6 +611,8 @@
  
  
  void RBBITest::TestJapaneseWordBreak() {
 +// TODO: Rewrite this test for a dictionary-based word breaking.
 +#if 0
      UErrorCode status = U_ZERO_ERROR;
      BITestData   japaneseWordSelection(status);
  
-@@ -591,6 +597,7 @@
+@@ -628,6 +634,7 @@
  
      generalIteratorTest(*e, japaneseWordSelection);
      delete e;
 +#endif
  }
  
  void RBBITest::TestTrieDict() {
-@@ -812,6 +819,372 @@
+@@ -849,6 +856,372 @@
      delete compact2;
  }
  
 +/*TODO: delete later*/
 +inline void writeEnumerationToFile(StringEnumeration *enumer, char *filename){
 +    UErrorCode      status  = U_ZERO_ERROR;
 +    FILE *outfile = fopen(filename,"w");
 +    UConverter *cvt = ucnv_open("UTF-8", &status);
 +    if (U_FAILURE(status))
 +        return;
@@ skipping 352 matching lines @@
 +    delete cloneEnum;
 +    delete compact2;
 +    utext_close(originalText);
 +    utext_close(cloneText);
 +
 + 
 +}
  
  //----------------------------------------------------------------------------
  //
-@@ -1832,8 +2205,15 @@
+@@ -1870,8 +2243,15 @@
  // Don't break in runs of hiragana or runs of ideograph, where the latter includes \u3005 \u3007 \u303B (cldrbug #2009).
  static const char    jaWordText[]     = "\\u79C1\\u9054\\u306B\\u4E00\\u3007\\u3007\\u3007\\u306E\\u30B3\\u30F3\\u30D4\\u30E5\\u30FC\\u30BF"
                                          "\\u304C\\u3042\\u308B\\u3002\\u5948\\u3005\\u306F\\u30EF\\u30FC\\u30C9\\u3067\\u3042\\u308B\\u3002";
 +#if 0 
  static const int32_t jaWordTOffsets[] = {    2, 3,          7, 8, 14,         17, 18,     20, 21, 24,         27, 28 };
  static const int32_t jaWordROffsets[] = { 1, 2, 3, 4, 5, 6, 7, 8, 14, 15, 16, 17, 18, 19, 20, 21, 24, 25, 26, 27, 28 };
 +#endif
 +// There's no separate Japanese word break iterator. Root is the same as Japanese.
 +// Our dictionary-based iterator has to be tweaked to better handle U+3005,
 +// U+3007, U+300B and some other cases.
 +static const int32_t jaWordTOffsets[] = { 1, 2, 3, 4, 5,    7, 8, 12, 13, 14, 15, 17, 18, 20, 21, 22, 23, 24, 25, 27, 28 };
 +static const int32_t jaWordROffsets[] = { 1, 2, 3, 4, 5,    7, 8, 12, 13, 14, 15, 17, 18, 20, 21, 22, 23, 24, 25, 27, 28 };
  
  // UBreakIteratorType UBRK_SENTENCE, Locale "el"
  // Add break after Greek question mark (cldrbug #2069).
-@@ -2580,6 +2960,8 @@
+@@ -2672,6 +3052,8 @@
      UnicodeSet  *fNewlineSet;
      UnicodeSet  *fKatakanaSet;
      UnicodeSet  *fALetterSet;
 +    // TODO(jungshik): Do we still need this change? 
 +    // UnicodeSet  *fALetterSet; // matches ALetterPlus in word.txt
      UnicodeSet  *fMidNumLetSet;
      UnicodeSet  *fMidLetterSet;
      UnicodeSet  *fMidNumSet;
-@@ -2588,6 +2970,7 @@
+@@ -2680,6 +3062,7 @@
      UnicodeSet  *fOtherSet;
      UnicodeSet  *fExtendSet;
      UnicodeSet  *fExtendNumLetSet;
 +    UnicodeSet  *fDictionaryCjkSet;
  
      RegexMatcher  *fMatcher;
  
-@@ -2604,12 +2987,24 @@
+@@ -2696,12 +3079,24 @@
      fCRSet           = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = CR}]"),           status);
      fLFSet           = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = LF}]"),           status);
      fNewlineSet      = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = Newline}]"),      status);
 -    fALetterSet      = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = ALetter}]"),      status);
 +    fDictionaryCjkSet= new UnicodeSet("[[\\uac00-\\ud7a3][:Han:][:Hiragana:]]", status);
 +    // Exclude Hangul syllables from ALetterSet during testing.
 +    // Leave CJK dictionary characters out from the monkey tests!
 +#if 0 
 +    fALetterSet      = new UnicodeSet("[\\p{Word_Break = ALetter}"
 +                                      "[\\p{Line_Break = Complex_Context}"
 +                                      "-\\p{Grapheme_Cluster_Break = Extend}"
 +                                      "-\\p{Grapheme_Cluster_Break = Control}"
 +                                      "]]",
 +                                      status);
 +#endif
 +    fALetterSet      = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = ALetter}]"), status);
 +    fALetterSet->removeAll(*fDictionaryCjkSet);
      fKatakanaSet     = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = Katakana}]"),     status);
      fMidNumLetSet    = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = MidNumLet}]"),    status);
      fMidLetterSet    = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = MidLetter}]"),    status);
      fMidNumSet       = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = MidNum}]"),       status);
 -    fNumericSet      = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = Numeric}]"),      status);
 +    fNumericSet      = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = Numeric}[\\uff10-\\uff19]]"),      status);
      fFormatSet       = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = Format}]"),       status);
      fExtendNumLetSet = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = ExtendNumLet}]"), status);
      fExtendSet       = new UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{Word_Break = Extend}]"),       status);
-@@ -2633,13 +3028,14 @@
+@@ -2725,13 +3120,14 @@
      fOtherSet->removeAll(*fFormatSet);
      fOtherSet->removeAll(*fExtendSet);
      // Inhibit dictionary characters from being tested at all.
 +    fOtherSet->removeAll(*fDictionaryCjkSet);
      fOtherSet->removeAll(UnicodeSet(UNICODE_STRING_SIMPLE("[\\p{LineBreak = Complex_Context}]"), status));
  
      fSets->addElement(fCRSet,        status);
      fSets->addElement(fLFSet,        status);
      fSets->addElement(fNewlineSet,   status);
      fSets->addElement(fALetterSet,   status);
 -    fSets->addElement(fKatakanaSet,  status);
 +    //fSets->addElement(fKatakanaSet,  status); //TODO: work out how to test katakana
      fSets->addElement(fMidLetterSet, status);
      fSets->addElement(fMidNumLetSet, status);
      fSets->addElement(fMidNumSet,    status);
-@@ -3871,6 +4267,7 @@
+@@ -3978,6 +4374,7 @@
      for (i = bi->last(); i != BreakIterator::DONE; i = bi->previous()) {
          count --;
          if (forward[count] != i) {
 +            printStringBreaks(ustr, expected, expectedcount);
              test->errln("happy break test previous() failed: expected %d but got %d",
                          forward[count], i);
              break;
-@@ -3904,23 +4301,25 @@
+@@ -4011,23 +4408,25 @@
      UErrorCode    status = U_ZERO_ERROR;
      // BreakIterator  *bi = BreakIterator::createCharacterInstance(locale, status);
      BreakIterator *bi = BreakIterator::createWordInstance(locale, status);
 +    // Replaced any C+J characters in a row with a random sequence of characters
 +    // of the same length to make our C+J segmentation not get in the way.
      static const char *strlist[] =
      {
      "\\U000e0032\\u0097\\u0f94\\uc2d8\\u05f4\\U000e0031\\u060d",
 -    "\\U000e0037\\u4666\\u1202\\u003a\\U000e0031\\u064d\\u0bea\\u591c\\U000e0040\\u003b",
 +    "\\U000e0037\\u2666\\u1202\\u003a\\U000e0031\\u064d\\u0bea\\u091c\\U000e0040\\u003b",
      "\\u0589\\u3e99\\U0001d7f3\\U000e0074\\u1810\\u200e\\U000e004b\\u0027\\U000e0061\\u003a",
      "\\u398c\\U000104a5\\U0001d173\\u102d\\u002e\\uca3b\\u002e\\u002c\\u5622",
 -    "\\u90ca\\u3588\\u009c\\u0953\\u194b",
 +    "\\uac00\\u3588\\u009c\\u0953\\u194b",
      "\\u200e\\U000e0072\\u0a4b\\U000e003f\\ufd2b\\u2027\\u002e\\u002e",
      "\\u0602\\u2019\\ua191\\U000e0063\\u0a4c\\u003a\\ub4b5\\u003a\\u827f\\u002e",
 -    "\\u7f1f\\uc634\\u65f8\\u0944\\u04f2\\uacdf\\u1f9c\\u05f4\\u002e",
 +    "\\u2f1f\\u1634\\u05f8\\u0944\\u04f2\\u0cdf\\u1f9c\\u05f4\\u002e",
      "\\U000e0042\\u002e\\u0fb8\\u09ef\\u0ed1\\u2044",
      "\\u003b\\u024a\\u102e\\U000e0071\\u0600",
      "\\u2027\\U000e0067\\u0a47\\u00b7",
      "\\u1fcd\\u002c\\u07aa\\u0027\\u11b0",
      "\\u002c\\U000e003c\\U0001d7f4\\u003a\\u0c6f\\u0027",
      "\\u0589\\U000e006e\\u0a42\\U000104a5",
 -    "\\u4f66\\ub523\\u003a\\uacae\\U000e0047\\u003a",
 +    "\\u0f66\\u2523\\u003a\\u0cae\\U000e0047\\u003a",
      "\\u003a\\u0f21\\u0668\\u0dab\\u003a\\u0655\\u00b7",
      "\\u0027\\u11af\\U000e0057\\u0602",
      "\\U0001d7f2\\U000e007\\u0004\\u0589",
-@@ -3932,7 +4331,7 @@
+@@ -4039,7 +4438,7 @@
      "\\u0be8\\u002e\\u0c68\\u066e\\u136d\\ufc99\\u59e7",
      "\\u0233\\U000e0020\\u0a69\\u0d6a",
      "\\u206f\\u0741\\ub3ab\\u2019\\ubcac\\u2019",
 -    "\\u58f4\\U000e0049\\u20e7\\u2027",
 +    "\\u18f4\\U000e0049\\u20e7\\u2027",
      "\\ub315\\U0001d7e5\\U000e0073\\u0c47\\u06f2\\u0c6a\\u0037\\u10fe",
      "\\ua183\\u102d\\u0bec\\u003a",
      "\\u17e8\\u06e7\\u002e\\u096d\\u003b",
-@@ -3942,7 +4341,7 @@
+@@ -4049,7 +4448,7 @@
      "\\U000e005d\\u2044\\u0731\\u0650\\u0061",
      "\\u003a\\u0664\\u00b7\\u1fba",
      "\\u003b\\u0027\\u00b7\\u47a3",
 -    "\\u2027\\U000e0067\\u0a42\\u00b7\\ubddf\\uc26c\\u003a\\u4186\\u041b",
 +    "\\u2027\\U000e0067\\u0a42\\u00b7\\u4edf\\uc26c\\u003a\\u4186\\u041b",
      "\\u0027\\u003a\\U0001d70f\\U0001d7df\\ubf4a\\U0001d7f5\\U0001d177\\u003a\\u0e51\\u1058\\U000e0058\\u00b7\\u0673",
      "\\uc30d\\u002e\\U000e002c\\u0c48\\u003a\\ub5a1\\u0661\\u002c",
      };
-@@ -3997,12 +4396,12 @@
+@@ -4104,12 +4503,12 @@
      "\\U0001d7f2\\U000e007d\\u0004\\u0589",
      "\\u82ab\\u17e8\\u0736\\u2019\\U0001d64d",
      "\\u0e01\\ub55c\\u0a68\\U000e0037\\u0cd6\\u002c\\ub959",
 -    "\\U000e0065\\u302c\\uc986\\u09ee\\U000e0068",
 +    "\\U000e0065\\u302c\\u09ee\\U000e0068",
      "\\u0be8\\u002e\\u0c68\\u066e\\u136d\\ufc99\\u59e7",
      "\\u0233\\U000e0020\\u0a69\\u0d6a",
      "\\u206f\\u0741\\ub3ab\\u2019\\ubcac\\u2019",
      "\\u58f4\\U000e0049\\u20e7\\u2027",
 -    "\\ub315\\U0001d7e5\\U000e0073\\u0c47\\u06f2\\u0c6a\\u0037\\u10fe",
 +    "\\U0001d7e5\\U000e0073\\u0c47\\u06f2\\u0c6a\\u0037\\u10fe",
      "\\ua183\\u102d\\u0bec\\u003a",
      "\\u17e8\\u06e7\\u002e\\u096d\\u003b",
      "\\u003a\\u0e57\\u0fad\\u002e",
---- source/test/intltest/rbbitst.h      2009-04-22 00:53:50.000000000 -0700
-+++ source/test/intltest/rbbitst.h      2009-07-27 13:01:17.767342000 -0700
+--- source/test/intltest/rbbitst.h      2010-07-22 17:15:37.000000000 -0700
++++ source/test/intltest/rbbitst.h      2011-01-21 14:12:45.152007000 -0800
 @@ -70,6 +70,7 @@
      void TestBug5775();
      void TestThaiBreaks();
      void TestTailoredBreaks();
 +    void TestTrieDictWithValue();
+     void TestDictRules();
+     void TestBug5532();
  
-     void TestDebug();
- 
---- source/test/testdata/rbbitst.txt    2009-06-24 14:06:38.000000000 -0700
-+++ source/test/testdata/rbbitst.txt    2009-07-29 12:56:31.483710000 -0700
-@@ -162,7 +162,23 @@
+--- source/test/testdata/rbbitst.txt    2010-07-28 17:18:28.000000000 -0700
++++ source/test/testdata/rbbitst.txt    2011-01-21 14:12:45.221011000 -0800
+@@ -161,7 +161,23 @@
  <data>•abc<200>\U0001D800•def<200>\U0001D3FF• •</data>
  
  # Hiragana & Katakana stay together, but separates from each other and Latin.
 -<data>•abc<200>\N{HIRAGANA LETTER SMALL A}<300>\N{HIRAGANA LETTER VU}\N{COMBINING ACUTE ACCENT}<300>\N{HIRAGANA ITERATION MARK}<300>\N{KATAKANA LETTER SMALL A}\N{KATAKANA ITERATION MARK}\N{HALFWIDTH KATAKANA LETTER WO}\N{HALFWIDTH KATAKANA LETTER N}<300>def<200>#•</data>
 +# *** what to do about theoretical combos of chars? i.e. hiragana + accent
 +#<data>•abc<200>\N{HIRAGANA LETTER SMALL A}<300>\N{HIRAGANA LETTER VU}\N{COMBINING ACUTE ACCENT}<300>\N{HIRAGANA ITERATION MARK}<300>\N{KATAKANA LETTER SMALL A}\N{KATAKANA ITERATION MARK}\N{HALFWIDTH KATAKANA LETTER WO}\N{HALFWIDTH KATAKANA LETTER N}<300>def<200>#•</data>
 +
 +# test normalization/dictionary handling of halfwidth katakana: same dictionary phrase in fullwidth and halfwidth
 +<data>•芽キャベツ<400>芽キャﾍﾞツ<400></data>
 +
 +# more Japanese tests
 +# TODO: Currently, U+30FC and other characters (script=common) in the Hiragana
 +# and the Katakana block are not treated correctly. Enable this later.
 +#<data>•どー<400>せ<400>日本語<400>を<400>勉強<400>する<400>理由<400>について<400>　•て<400>こと<400>は<400>我<400>でも<400>知<400>ら<400>も<400>い<400>こと<400>なん<400>だ<400>。•</data>
 +<data>•日本語<400>を<400>勉強<400>する<400>理由<400>について<400>　•て<400>こと<400>は<400>我<400>でも<400>知<400>ら<400>も<400>い<400>こと<400>なん<400>だ<400>。•</data>
 +
 +# Testing of word boundary for dictionary word containing both kanji and kana
 +<data>•中だるみ<400>蔵王の森<400>ウ離島<400></data>
 +
 +# Testing of Chinese segmentation (taken from a Chinese news article)
 +<data>•400<100>余<400>名<400>中央<400>委员<400>和<400>中央<400>候补<400>委员<400>都<400>领<400>到了<400>“•推荐<400>票<400>”•，•有<400>资格<400>在<400>200<100>多<400>名<400>符合<400>条件<400>的<400>63<100>岁<400>以下<400>中共<400>正<400>部<400>级<400>干部<400>中<400>，•选出<400>他们<400>属意<400>的<400>中央<400>政治局<400>委员<400>以<400>向<400>政治局<400>常委<400>会<400>举荐<400>。•</data>
  
  # Words with interior formatting characters
  <data>•def\N{COMBINING ACUTE ACCENT}\N{SYRIAC ABBREVIATION MARK}ghi<200> •</data>
-@@ -170,6 +186,8 @@
+@@ -169,6 +185,8 @@
  # to test for bug #4097779
  <data>•aa\N{COMBINING GRAVE ACCENT}a<200> •</data>
  
 +# fullwidth numeric, midletter characters etc should be treated like their halfwidth counterparts
 +<data>•ＩＳＮ'Ｔ<200> •１９<100>日<400></data>
  
  #      to test for bug #4098467
  #      What follows is a string of Korean characters (I found it in the Yellow Pages
-@@ -179,9 +197,15 @@
+@@ -178,9 +196,15 @@
  #      precomposed syllables...
  <data>•\uc0c1\ud56d<200> •\ud55c\uc778<200> •\uc5f0\ud569<200> •\uc7a5\ub85c\uad50\ud68c<200> •\u1109\u1161\u11bc\u1112\u1161\u11bc<200> •\u1112\u1161\u11ab\u110b\u1175\u11ab<200> •\u110b\u1167\u11ab\u1112\u1161\u11b8<200> •\u110c\u1161\u11bc\u1105\u1169\u1100\u116d\u1112\u116c<200> •</data>
  
 -<data>•abc<200>\u4e01<400>\u4e02<400>\u3005<200>\u4e03<400>\u4e03<400>abc<200> •</data>
 +# more Korean tests (Jamo not tested here, not counted as dictionary characters)
 +# Disable them now because we don't include a Korean dictionary.
 +#<data>•\ud55c\uad6d<200>\ub300\ud559\uad50<200>\uc790\uc5f0<200>\uacfc\ud559<200>\ub300\ud559<200>\ubb3c\ub9ac\ud559\uacfc<200></data>
 +#<data>•\ud604\uc7ac<200>\ub294<200> •\uac80\ucc30<200>\uc774<200> •\ubd84\uc2dd<200>\ud68c\uacc4<200>\ubb38\uc81c<200>\ub97c<200> •\uc870\uc0ac<200>\ud560<200> •\uac00\ub2a5\uc131<200>\uc740<200> •\uc5c6\ub2e4<200>\u002e•</data>
 +
 +<data>•abc<200>\u4e01<400>\u4e02<400>\u3005<400>\u4e03\u4e03<400>abc<200> •</data>
 +
 +<data>•\u06c9<200>\uc799<200>\ufffa•</data>
  
 -<data>•\u06c9\uc799\ufffa<200></data>
  
  #      
  #      Try some words from other scripts.
-@@ -492,8 +516,7 @@
+@@ -491,8 +515,7 @@
  <data>•\uc0c1•\ud56d •\ud55c•\uc778 •\uc5f0•\ud569 •\uc7a5•\ub85c•\uad50•\ud68c•</data>
  
  #      conjoining jamo...
 -#      TODO:  rules update needed
 -#<data>•\u1109\u1161\u11bc•\u1112\u1161\u11bc •\u1112\u1161\u11ab•\u110b\u1175\u11ab #•\u110b\u1167\u11ab•\u1112\u1161\u11b8 •\u110c\u1161\u11bc•\u1105\u1169•\u1100\u116d•\u1112\u116c•</data>
 +<data>•\u1109\u1161\u11bc•\u1112\u1161\u11bc •\u1112\u1161\u11ab•\u110b\u1175\u11ab •\u110b\u1167\u11ab•\u1112\u1161\u11b8 •\u110c\u1161\u11bc•\u1105\u1169•\u1100\u116d•\u1112\u116c•</data>
  
  #      to test for bug #4117554: Fullwidth .!? should be treated as postJwrd
  <data>•\u4e01\uff0e•\u4e02\uff01•\u4e03\uff1f•</data>
---- source/test/testdata/testaliases.txt        2009-06-24 14:06:38.000000000 -0700
-+++ source/test/testdata/testaliases.txt        2009-07-28 17:07:26.251120000 -0700
+--- source/test/testdata/testaliases.txt        2009-11-12 13:53:42.000000000 -0800
++++ source/test/testdata/testaliases.txt        2011-01-21 14:12:45.204005000 -0800
 @@ -28,7 +28,7 @@
      LocaleScript:alias { "/ICUDATA/ja/LocaleScript" }
  
      // aliasing using position
 -    boundaries:alias { "/ICUDATA-brkitr/ja" } // Referencing corresponding resource in another bundle
 +    boundaries:alias { "/ICUDATA-brkitr/th" } // Referencing corresponding resource in another bundle
  
      // aliasing arrays
      zoneTests {
---- source/tools/genctd/genctd.cpp      2006-09-04 09:28:24.000000000 -0700
-+++ source/tools/genctd/genctd.cpp      2009-07-27 13:01:17.776335000 -0700
+--- source/tools/genctd/genctd.cpp      2009-08-04 14:09:17.000000000 -0700
++++ source/tools/genctd/genctd.cpp      2011-01-21 14:12:45.564923000 -0800
 @@ -1,6 +1,6 @@
  /*
  **********************************************************************
--*   Copyright (C) 2002-2006, International Business Machines
-+*   Copyright (C) 2002-2006,2008, International Business Machines
+-*   Copyright (C) 2002-2009, International Business Machines
++*   Copyright (C) 2002-2010, International Business Machines
  *   Corporation and others.  All Rights Reserved.
  **********************************************************************
  *
 @@ -34,12 +34,15 @@
  #include "unicode/udata.h"
  #include "unicode/putil.h"
  
 +//#include "unicode/ustdio.h"
 +
  #include "uoptions.h"
  #include "unewdata.h"
  #include "ucmndata.h"
  #include "rbbidata.h"
  #include "triedict.h"
  #include "cmemory.h"
 +#include "uassert.h"
  
  #include <stdio.h>
  #include <stdlib.h>
-@@ -198,147 +201,191 @@
+@@ -199,147 +202,191 @@
      long        wordFileSize;
      FILE        *file;
      char        *wordBufferC;
 -
 +    MutableTrieDictionary *mtd = NULL;
 +    
      file = fopen(wordFileName, "rb");
 -    if( file == 0 ) {
 -        fprintf(stderr, "Could not open file \"%s\"\n", wordFileName);
 -        exit(-1);
@@ skipping 301 matching lines @@
 +        // Get rid of the Unicode text buffer
 +        delete[] wordSourceU;
      }
  
 -    // Get rid of the Unicode text buffer
 -    delete[] wordSourceU;
 -
      // Now, create a CompactTrieDictionary from the mutable dictionary
      CompactTrieDictionary *ctd = new CompactTrieDictionary(*mtd, status);
      if (U_FAILURE(status)) {
-@@ -392,4 +439,3 @@
+@@ -393,4 +440,3 @@
  
  #endif /* #if !UCONFIG_NO_BREAK_ITERATION */
  }
 -
 --- source/tools/genctd/Makefile.in     2006-12-16 13:07:01.000000000 -0800
-+++ source/tools/genctd/Makefile.in     2009-07-27 13:01:17.782326000 -0700
++++ source/tools/genctd/Makefile.in     2011-01-21 14:12:45.555920000 -0800
 @@ -23,13 +23,13 @@
  ## Extra files to remove for 'make clean'
  CLEANFILES = *~ $(DEPS) $(MAN_FILES)
  
 -## Target information
 +## Target informationcd 
  TARGET = $(BINDIR)/$(TARGET_STUB_NAME)$(EXEEXT)
  
  ifneq ($(top_builddir),$(top_srcdir))
  CPPFLAGS += -I$(top_builddir)/common
  endif
 -CPPFLAGS += -I$(top_srcdir)/common -I$(srcdir)/../toolutil
 +CPPFLAGS += -I$(top_srcdir)/common -I$(srcdir)/../toolutil -I$(top_srcdir)/i18n
  LIBS = $(LIBICUTOOLUTIL) $(LIBICUI18N) $(LIBICUUC) $(DEFAULT_LIBS) $(LIB_M)
  
  OBJECTS = genctd.o
---- source/data/Makefile.in     2009-05-20 23:03:54.000000000 -0700
-+++ source/data/Makefile.in     2009-10-21 15:43:18.235201000 -0700
-@@ -452,8 +452,9 @@
- ####################################################    CTD
- # CTD FILES
- 
--$(BRKBLDDIR)/%.ctd: $(BRKSRCDIR)/%.txt $(TOOLBINDIR)/genctd$(TOOLEXEEXT) $(DAT_FILES)
--       $(INVOKE) $(TOOLBINDIR)/genctd -c -i $(BUILDDIR) -o $@ $<
-+# .ctd file now generated regardless of whether dictionary file exists
-+$(BRKBLDDIR)/%.ctd: $(TOOLBINDIR)/genctd$(TOOLEXEEXT) $(DAT_FILES)
-+       $(INVOKE) $(TOOLBINDIR)/genctd -c -i $(BUILDDIR) -o $@ $(BRKSRCDIR)/$(*F).txt
- 
- ####################################################    CFU
- # CFU FILES