ICU 56 update step 1

source/i18n/collationdatareader.h

 /*
 *******************************************************************************
-* Copyright (C) 2013-2014, International Business Machines
+* Copyright (C) 2013-2015, International Business Machines
 * Corporation and others.  All Rights Reserved.
 *******************************************************************************
 * collationdatareader.h
 *
 * created on: 2013feb07
 * created by: Markus W. Scherer
 */
 
 #ifndef __COLLATIONDATAREADER_H__
 #define __COLLATIONDATAREADER_H__
@@ skipping 88 matching lines @@
 
     static UBool U_CALLCONV
     isAcceptable(void *context, const char *type, const char *name, const UDataInfo *pInfo);
 
 private:
     CollationDataReader();  // no constructor
 };
 
 /*
  * Format of collation data (ucadata.icu, binary data in coll/ *.res files).
- * Format version 4.0.
+ * Format version 5.
  *
  * The root collation data is stored in the ucadata.icu file.
  * Tailorings are stored inside .res resource bundle files, with a complete file header.
  *
  * Collation data begins with a standard ICU data file header
  * (DataHeader, see ucmndata.h and unicode/udata.h).
  * The UDataInfo.dataVersion field contains the UCA and other version numbers,
  * see the comments for CollationTailoring.version.
  *
  * After the header, the file contains the following parts.
@@ skipping 21 matching lines @@
  *      Byte offsets are offsets from after the header,
  *      that is, from the beginning of the indexes[].
  *      Each part starts at an offset with proper alignment for its data.
  *      If necessary, the previous part may include padding bytes to achieve this alignment.
  *      The last byte offset that is stored in the indexes indicates the total size of the data
  *      (starting with the indexes).
  *
  * int32_t reorderCodes[]; -- empty in root
  *      The list of script and reordering codes.
  *
+ *      Beginning with format version 5, this array may optionally
+ *      have trailing entries with a full list of reorder ranges
+ *      as described for CollationSettings::reorderRanges.
+ *
+ *      Script or reorder codes are first and do not exceed 16-bit values.
+ *      Range limits are stored in the upper 16 bits, and are never 0.
+ *      Split this array into reorder codes and ranges at the first entry
+ *      with non-zero upper 16 bits.
+ *
+ *      If the ranges are missing but needed for split-reordered primary lead bytes,
+ *      then they are regenerated at load time.
+ *
  * uint8_t reorderTable[256]; -- empty in root; can be longer to include padding bytes
  *      Primary-weight lead byte permutation table.
  *      Normally present when the reorderCodes are, but can be built at load time.
  *
+ *      Beginning with format version 5, a 0 entry at a non-zero index
+ *      (which is otherwise an illegal value)
+ *      means that the primary lead byte is "split"
+ *      (there are different offsets for primaries that share that lead byte)
+ *      and the reordering offset must be determined via the reorder ranges
+ *      that are either stored as part of the reorderCodes array
+ *      or regenerated at load time.
+ *
  * UTrie2 trie; -- see utrie2_impl.h and utrie2.h
  *      The trie holds the main collation data. Each code point is mapped to a 32-bit value.
  *      It encodes a simple collation element (CE) in compact form, unless bits 7..6 are both set,
  *      in which case it is a special CE32 and contains a 4-bit tag and further data.
  *      See the Collation class for details.
  *
  *      The trie has a value for each lead surrogate code unit with some bits encoding
  *      collective properties of the 1024 supplementary characters whose UTF-16 form starts with
  *      the lead surrogate. See Collation::LEAD_SURROGATE_TAG..
  *
@@ skipping 19 matching lines @@
  *      code points are unsafe.
  *      Does not include [:^lccc=0:][:^tccc=0:].
  *      For each tailoring, the root unsafeBackwardSet is subtracted.
  *      (As a result, in many tailorings no set needs to be stored.)
  *
  * uint16_t fastLatinTable[];
  *      Optional optimization for Latin text.
  *      See the CollationFastLatin class.
  *
  * uint16_t scripts[]; -- empty in all tailorings
+ *      Format version 5:
+ *      uint16_t numScripts;
+ *      uint16_t scriptsIndex[numScripts+16];
+ *      uint16_t scriptStarts[];
+ *      See CollationData::numScripts etc.
+ *
+ *      Format version 4:
  *      Table of the reordering groups with their first and last lead bytes,
  *      and their script and reordering codes.
  *      See CollationData::scripts.
  *
  * UBool compressibleBytes[]; -- empty in all tailorings
  *      Flag for getSortKey(), indicating primary weight lead bytes that are compressible.
+ *
+ * -----------------
+ * Changes for formatVersion 5 (ICU 55)
+ *
+ * Reordering moves single scripts, not groups of scripts.
+ * Reorder ranges are optionally appended to the reorderCodes,
+ * and a 0 entry in the reorderTable indicates a split lead byte.
+ * The scripts data has a new format.
+ *
+ * The rootElements may contain secondary and tertiary weights below common=05.
+ * (Used for small Hiragana letters.)
+ * Where is occurs, there is also an explicit unit with common secondary & tertiary weights.
+ * There are no other data structure changes, but builder code needs to be able to handle such data.
+ *
+ * The collation element for the merge separator code point U+FFFE
+ * does not necessarily have special, unique secondary/tertiary weights any more.
  */
 
 U_NAMESPACE_END
 
 #endif  // !UCONFIG_NO_COLLATION
 #endif  // __COLLATIONDATAREADER_H__