ICU 56 update step 1

source/i18n/collationdatareader.cpp

 /*
 *******************************************************************************
-* Copyright (C) 2013-2014, International Business Machines
+* Copyright (C) 2013-2015, International Business Machines
 * Corporation and others.  All Rights Reserved.
 *******************************************************************************
 * collationdatareader.cpp
 *
 * created on: 2013feb07
 * created by: Markus W. Scherer
 */
 
 #include "unicode/utypes.h"
 
 #if !UCONFIG_NO_COLLATION
 
 #include "unicode/ucol.h"
 #include "unicode/udata.h"
 #include "unicode/uscript.h"
 #include "cmemory.h"
 #include "collation.h"
 #include "collationdata.h"
 #include "collationdatareader.h"
 #include "collationfastlatin.h"
 #include "collationkeys.h"
 #include "collationrootelements.h"
 #include "collationsettings.h"
 #include "collationtailoring.h"
+#include "collunsafe.h"
 #include "normalizer2impl.h"
 #include "uassert.h"
 #include "ucmndata.h"
 #include "utrie2.h"
 
 U_NAMESPACE_BEGIN
 
 namespace {
 
 int32_t getIndex(const int32_t *indexes, int32_t length, int32_t i) {
@@ skipping 57 matching lines @@
         length = 0;  // only indexes, and inLength was already checked for them
     }
     if(0 <= inLength && inLength < length) {
         errorCode = U_INVALID_FORMAT_ERROR;
         return;
     }
 
     const CollationData *baseData = base == NULL ? NULL : base->data;
     const int32_t *reorderCodes = NULL;
     int32_t reorderCodesLength = 0;
+    const uint32_t *reorderRanges = NULL;
+    int32_t reorderRangesLength = 0;
     index = IX_REORDER_CODES_OFFSET;
     offset = getIndex(inIndexes, indexesLength, index);
     length = getIndex(inIndexes, indexesLength, index + 1) - offset;
     if(length >= 4) {
         if(baseData == NULL) {
             // We assume for collation settings that
             // the base data does not have a reordering.
             errorCode = U_INVALID_FORMAT_ERROR;
             return;
         }
         reorderCodes = reinterpret_cast<const int32_t *>(inBytes + offset);
         reorderCodesLength = length / 4;
+
+        // The reorderRanges (if any) are the trailing reorderCodes entries.
+        // Split the array at the boundary.
+        // Script or reorder codes do not exceed 16-bit values.
+        // Range limits are stored in the upper 16 bits, and are never 0.
+        while(reorderRangesLength < reorderCodesLength &&
+                (reorderCodes[reorderCodesLength - reorderRangesLength - 1] & 0xffff0000) != 0) {
+            ++reorderRangesLength;
+        }
+        U_ASSERT(reorderRangesLength < reorderCodesLength);
+        if(reorderRangesLength != 0) {
+            reorderCodesLength -= reorderRangesLength;
+            reorderRanges = reinterpret_cast<const uint32_t *>(reorderCodes + reorderCodesLength);
+        }
     }
 
     // There should be a reorder table only if there are reorder codes.
     // However, when there are reorder codes the reorder table may be omitted to reduce
     // the data size.
     const uint8_t *reorderTable = NULL;
     index = IX_REORDER_TABLE_OFFSET;
     offset = getIndex(inIndexes, indexesLength, index);
     length = getIndex(inIndexes, indexesLength, index + 1) - offset;
     if(length >= 256) {
@@ skipping 112 matching lines @@
 
     index = IX_UNSAFE_BWD_OFFSET;
     offset = getIndex(inIndexes, indexesLength, index);
     length = getIndex(inIndexes, indexesLength, index + 1) - offset;
     if(length >= 2) {
         if(data == NULL) {
             errorCode = U_INVALID_FORMAT_ERROR;
             return;
         }
         if(baseData == NULL) {
+#if defined(COLLUNSAFE_COLL_VERSION) && defined (COLLUNSAFE_SERIALIZE)
+          tailoring.unsafeBackwardSet = new UnicodeSet(unsafe_serializedData, unsafe_serializedCount, UnicodeSet::kSerialized, errorCode);
+          if(tailoring.unsafeBackwardSet == NULL) {
+            errorCode = U_MEMORY_ALLOCATION_ERROR;
+            return;
+          } else if (U_FAILURE(errorCode)) {
+            return;
+          }
+#else
             // Create the unsafe-backward set for the root collator.
             // Include all non-zero combining marks and trail surrogates.
             // We do this at load time, rather than at build time,
             // to simplify Unicode version bootstrapping:
             // The root data builder only needs the new FractionalUCA.txt data,
             // but it need not be built with a version of ICU already updated to
             // the corresponding new Unicode Character Database.
             //
             // The following is an optimized version of
             // new UnicodeSet("[[:^lccc=0:][\\udc00-\\udfff]]").
             // It is faster and requires fewer code dependencies.
             tailoring.unsafeBackwardSet = new UnicodeSet(0xdc00, 0xdfff);  // trail surrogates
             if(tailoring.unsafeBackwardSet == NULL) {
                 errorCode = U_MEMORY_ALLOCATION_ERROR;
                 return;
             }
             data->nfcImpl.addLcccChars(*tailoring.unsafeBackwardSet);
+#endif // !COLLUNSAFE_SERIALIZE || !COLLUNSAFE_COLL_VERSION
         } else {
             // Clone the root collator's set contents.
             tailoring.unsafeBackwardSet = static_cast<UnicodeSet *>(
                 baseData->unsafeBackwardSet->cloneAsThawed());
             if(tailoring.unsafeBackwardSet == NULL) {
                 errorCode = U_MEMORY_ALLOCATION_ERROR;
                 return;
             }
         }
         // Add the ranges from the data file to the unsafe-backward set.
@@ skipping 54 matching lines @@
     }
 
     index = IX_SCRIPTS_OFFSET;
     offset = getIndex(inIndexes, indexesLength, index);
     length = getIndex(inIndexes, indexesLength, index + 1) - offset;
     if(length >= 2) {
         if(data == NULL) {
             errorCode = U_INVALID_FORMAT_ERROR;
             return;
         }
-        data->scripts = reinterpret_cast<const uint16_t *>(inBytes + offset);
-        data->scriptsLength = length / 2;
+        const uint16_t *scripts = reinterpret_cast<const uint16_t *>(inBytes + offset);
+        int32_t scriptsLength = length / 2;
+        data->numScripts = scripts[0];
+        // There must be enough entries for both arrays, including more than two range starts.
+        data->scriptStartsLength = scriptsLength - (1 + data->numScripts + 16);
+        if(data->scriptStartsLength <= 2 ||
+                CollationData::MAX_NUM_SCRIPT_RANGES < data->scriptStartsLength) {
+            errorCode = U_INVALID_FORMAT_ERROR;
+            return;
+        }
+        data->scriptsIndex = scripts + 1;
+        data->scriptStarts = scripts + 1 + data->numScripts + 16;
+        if(!(data->scriptStarts[0] == 0 &&
+                data->scriptStarts[1] == ((Collation::MERGE_SEPARATOR_BYTE + 1) << 8) &&
+                data->scriptStarts[data->scriptStartsLength - 1] ==
+                        (Collation::TRAIL_WEIGHT_BYTE << 8))) {
+            errorCode = U_INVALID_FORMAT_ERROR;
+            return;
+        }
     } else if(data == NULL) {
         // Nothing to do.
     } else if(baseData != NULL) {
-        data->scripts = baseData->scripts;
-        data->scriptsLength = baseData->scriptsLength;
+        data->numScripts = baseData->numScripts;
+        data->scriptsIndex = baseData->scriptsIndex;
+        data->scriptStarts = baseData->scriptStarts;
+        data->scriptStartsLength = baseData->scriptStartsLength;
     }
 
     index = IX_COMPRESSIBLE_BYTES_OFFSET;
     offset = getIndex(inIndexes, indexesLength, index);
     length = getIndex(inIndexes, indexesLength, index + 1) - offset;
     if(length >= 256) {
         if(data == NULL) {
             errorCode = U_INVALID_FORMAT_ERROR;
             return;
         }
@@ skipping 29 matching lines @@
     }
     settings->options = options;
     // Set variableTop from options and scripts data.
     settings->variableTop = tailoring.data->getLastPrimaryForGroup(
             UCOL_REORDER_CODE_FIRST + settings->getMaxVariable());
     if(settings->variableTop == 0) {
         errorCode = U_INVALID_FORMAT_ERROR;
         return;
     }
 
-    if(reorderCodesLength == 0 || reorderTable != NULL) {
-        settings->aliasReordering(reorderCodes, reorderCodesLength, reorderTable);
-    } else {
-        uint8_t table[256];
-        baseData->makeReorderTable(reorderCodes, reorderCodesLength, table, errorCode);
-        if(U_FAILURE(errorCode)) { return; }
-        if(!settings->setReordering(reorderCodes, reorderCodesLength,table)) {
-            errorCode = U_MEMORY_ALLOCATION_ERROR;
-            return;
-        }
+    if(reorderCodesLength != 0) {
+        settings->aliasReordering(*baseData, reorderCodes, reorderCodesLength,
+                                  reorderRanges, reorderRangesLength,
+                                  reorderTable, errorCode);
     }
 
     settings->fastLatinOptions = CollationFastLatin::getOptions(
         tailoring.data, *settings,
         settings->fastLatinPrimaries, UPRV_LENGTHOF(settings->fastLatinPrimaries));
 }
 
 UBool U_CALLCONV
 CollationDataReader::isAcceptable(void *context,
                                   const char * /* type */, const char * /*name*/,
                                   const UDataInfo *pInfo) {
     if(
         pInfo->size >= 20 &&
         pInfo->isBigEndian == U_IS_BIG_ENDIAN &&
         pInfo->charsetFamily == U_CHARSET_FAMILY &&
         pInfo->dataFormat[0] == 0x55 &&  // dataFormat="UCol"
         pInfo->dataFormat[1] == 0x43 &&
         pInfo->dataFormat[2] == 0x6f &&
         pInfo->dataFormat[3] == 0x6c &&
-        pInfo->formatVersion[0] == 4
+        pInfo->formatVersion[0] == 5
     ) {
         UVersionInfo *version = static_cast<UVersionInfo *>(context);
         if(version != NULL) {
             uprv_memcpy(version, pInfo->dataVersion, 4);
         }
         return TRUE;
     } else {
         return FALSE;
     }
 }
 
 U_NAMESPACE_END
 
 #endif  // !UCONFIG_NO_COLLATION