Index: icu46/source/common/ucnvsel.cpp |
=================================================================== |
--- icu46/source/common/ucnvsel.cpp (revision 0) |
+++ icu46/source/common/ucnvsel.cpp (revision 0) |
@@ -0,0 +1,811 @@ |
+/* |
+******************************************************************************* |
+* |
+* Copyright (C) 2008-2009, International Business Machines |
+* Corporation, Google and others. All Rights Reserved. |
+* |
+******************************************************************************* |
+*/ |
+// Author : eldawy@google.com (Mohamed Eldawy) |
+// ucnvsel.cpp |
+// |
+// Purpose: To generate a list of encodings capable of handling |
+// a given Unicode text |
+// |
+// Started 09-April-2008 |
+ |
+/** |
+ * \file |
+ * |
+ * This is an implementation of an encoding selector. |
+ * The goal is, given a unicode string, find the encodings |
+ * this string can be mapped to. To make processing faster |
+ * a trie is built when you call ucnvsel_open() that |
+ * stores all encodings a codepoint can map to |
+ */ |
+ |
+#include "unicode/ucnvsel.h" |
+ |
+#include <string.h> |
+ |
+#include "unicode/uchar.h" |
+#include "unicode/uniset.h" |
+#include "unicode/ucnv.h" |
+#include "unicode/ustring.h" |
+#include "unicode/uchriter.h" |
+#include "utrie2.h" |
+#include "propsvec.h" |
+#include "uassert.h" |
+#include "ucmndata.h" |
+#include "uenumimp.h" |
+#include "cmemory.h" |
+#include "cstring.h" |
+ |
+U_NAMESPACE_USE |
+ |
+struct UConverterSelector { |
+ UTrie2 *trie; // 16 bit trie containing offsets into pv |
+ uint32_t* pv; // table of bits! |
+ int32_t pvCount; |
+ char** encodings; // which encodings did user ask to use? |
+ int32_t encodingsCount; |
+ int32_t encodingStrLength; |
+ uint8_t* swapped; |
+ UBool ownPv, ownEncodingStrings; |
+}; |
+ |
+static void generateSelectorData(UConverterSelector* result, |
+ UPropsVectors *upvec, |
+ const USet* excludedCodePoints, |
+ const UConverterUnicodeSet whichSet, |
+ UErrorCode* status) { |
+ if (U_FAILURE(*status)) { |
+ return; |
+ } |
+ |
+ int32_t columns = (result->encodingsCount+31)/32; |
+ |
+ // set errorValue to all-ones |
+ for (int32_t col = 0; col < columns; col++) { |
+ upvec_setValue(upvec, UPVEC_ERROR_VALUE_CP, UPVEC_ERROR_VALUE_CP, |
+ col, ~0, ~0, status); |
+ } |
+ |
+ for (int32_t i = 0; i < result->encodingsCount; ++i) { |
+ uint32_t mask; |
+ uint32_t column; |
+ int32_t item_count; |
+ int32_t j; |
+ UConverter* test_converter = ucnv_open(result->encodings[i], status); |
+ if (U_FAILURE(*status)) { |
+ return; |
+ } |
+ USet* unicode_point_set; |
+ unicode_point_set = uset_open(1, 0); // empty set |
+ |
+ ucnv_getUnicodeSet(test_converter, unicode_point_set, |
+ whichSet, status); |
+ if (U_FAILURE(*status)) { |
+ ucnv_close(test_converter); |
+ return; |
+ } |
+ |
+ column = i / 32; |
+ mask = 1 << (i%32); |
+ // now iterate over intervals on set i! |
+ item_count = uset_getItemCount(unicode_point_set); |
+ |
+ for (j = 0; j < item_count; ++j) { |
+ UChar32 start_char; |
+ UChar32 end_char; |
+ UErrorCode smallStatus = U_ZERO_ERROR; |
+ uset_getItem(unicode_point_set, j, &start_char, &end_char, NULL, 0, |
+ &smallStatus); |
+ if (U_FAILURE(smallStatus)) { |
+ // this will be reached for the converters that fill the set with |
+ // strings. Those should be ignored by our system |
+ } else { |
+ upvec_setValue(upvec, start_char, end_char, column, ~0, mask, |
+ status); |
+ } |
+ } |
+ ucnv_close(test_converter); |
+ uset_close(unicode_point_set); |
+ if (U_FAILURE(*status)) { |
+ return; |
+ } |
+ } |
+ |
+ // handle excluded encodings! Simply set their values to all 1's in the upvec |
+ if (excludedCodePoints) { |
+ int32_t item_count = uset_getItemCount(excludedCodePoints); |
+ for (int32_t j = 0; j < item_count; ++j) { |
+ UChar32 start_char; |
+ UChar32 end_char; |
+ |
+ uset_getItem(excludedCodePoints, j, &start_char, &end_char, NULL, 0, |
+ status); |
+ for (int32_t col = 0; col < columns; col++) { |
+ upvec_setValue(upvec, start_char, end_char, col, ~0, ~0, |
+ status); |
+ } |
+ } |
+ } |
+ |
+ // alright. Now, let's put things in the same exact form you'd get when you |
+ // unserialize things. |
+ result->trie = upvec_compactToUTrie2WithRowIndexes(upvec, status); |
+ result->pv = upvec_cloneArray(upvec, &result->pvCount, NULL, status); |
+ result->pvCount *= columns; // number of uint32_t = rows * columns |
+ result->ownPv = TRUE; |
+} |
+ |
+/* open a selector. If converterListSize is 0, build for all converters. |
+ If excludedCodePoints is NULL, don't exclude any codepoints */ |
+U_CAPI UConverterSelector* U_EXPORT2 |
+ucnvsel_open(const char* const* converterList, int32_t converterListSize, |
+ const USet* excludedCodePoints, |
+ const UConverterUnicodeSet whichSet, UErrorCode* status) { |
+ // check if already failed |
+ if (U_FAILURE(*status)) { |
+ return NULL; |
+ } |
+ // ensure args make sense! |
+ if (converterListSize < 0 || (converterList == NULL && converterListSize != 0)) { |
+ *status = U_ILLEGAL_ARGUMENT_ERROR; |
+ return NULL; |
+ } |
+ |
+ // allocate a new converter |
+ LocalUConverterSelectorPointer newSelector( |
+ (UConverterSelector*)uprv_malloc(sizeof(UConverterSelector))); |
+ if (newSelector.isNull()) { |
+ *status = U_MEMORY_ALLOCATION_ERROR; |
+ return NULL; |
+ } |
+ uprv_memset(newSelector.getAlias(), 0, sizeof(UConverterSelector)); |
+ |
+ if (converterListSize == 0) { |
+ converterList = NULL; |
+ converterListSize = ucnv_countAvailable(); |
+ } |
+ newSelector->encodings = |
+ (char**)uprv_malloc(converterListSize * sizeof(char*)); |
+ if (!newSelector->encodings) { |
+ *status = U_MEMORY_ALLOCATION_ERROR; |
+ return NULL; |
+ } |
+ newSelector->encodings[0] = NULL; // now we can call ucnvsel_close() |
+ |
+ // make a backup copy of the list of converters |
+ int32_t totalSize = 0; |
+ int32_t i; |
+ for (i = 0; i < converterListSize; i++) { |
+ totalSize += |
+ (int32_t)uprv_strlen(converterList != NULL ? converterList[i] : ucnv_getAvailableName(i)) + 1; |
+ } |
+ // 4-align the totalSize to 4-align the size of the serialized form |
+ int32_t encodingStrPadding = totalSize & 3; |
+ if (encodingStrPadding != 0) { |
+ encodingStrPadding = 4 - encodingStrPadding; |
+ } |
+ newSelector->encodingStrLength = totalSize += encodingStrPadding; |
+ char* allStrings = (char*) uprv_malloc(totalSize); |
+ if (!allStrings) { |
+ *status = U_MEMORY_ALLOCATION_ERROR; |
+ return NULL; |
+ } |
+ |
+ for (i = 0; i < converterListSize; i++) { |
+ newSelector->encodings[i] = allStrings; |
+ uprv_strcpy(newSelector->encodings[i], |
+ converterList != NULL ? converterList[i] : ucnv_getAvailableName(i)); |
+ allStrings += uprv_strlen(newSelector->encodings[i]) + 1; |
+ } |
+ while (encodingStrPadding > 0) { |
+ *allStrings++ = 0; |
+ --encodingStrPadding; |
+ } |
+ |
+ newSelector->ownEncodingStrings = TRUE; |
+ newSelector->encodingsCount = converterListSize; |
+ UPropsVectors *upvec = upvec_open((converterListSize+31)/32, status); |
+ generateSelectorData(newSelector.getAlias(), upvec, excludedCodePoints, whichSet, status); |
+ upvec_close(upvec); |
+ |
+ if (U_FAILURE(*status)) { |
+ return NULL; |
+ } |
+ |
+ return newSelector.orphan(); |
+} |
+ |
+/* close opened selector */ |
+U_CAPI void U_EXPORT2 |
+ucnvsel_close(UConverterSelector *sel) { |
+ if (!sel) { |
+ return; |
+ } |
+ if (sel->ownEncodingStrings) { |
+ uprv_free(sel->encodings[0]); |
+ } |
+ uprv_free(sel->encodings); |
+ if (sel->ownPv) { |
+ uprv_free(sel->pv); |
+ } |
+ utrie2_close(sel->trie); |
+ uprv_free(sel->swapped); |
+ uprv_free(sel); |
+} |
+ |
+static const UDataInfo dataInfo = { |
+ sizeof(UDataInfo), |
+ 0, |
+ |
+ U_IS_BIG_ENDIAN, |
+ U_CHARSET_FAMILY, |
+ U_SIZEOF_UCHAR, |
+ 0, |
+ |
+ { 0x43, 0x53, 0x65, 0x6c }, /* dataFormat="CSel" */ |
+ { 1, 0, 0, 0 }, /* formatVersion */ |
+ { 0, 0, 0, 0 } /* dataVersion */ |
+}; |
+ |
+enum { |
+ UCNVSEL_INDEX_TRIE_SIZE, // trie size in bytes |
+ UCNVSEL_INDEX_PV_COUNT, // number of uint32_t in the bit vectors |
+ UCNVSEL_INDEX_NAMES_COUNT, // number of encoding names |
+ UCNVSEL_INDEX_NAMES_LENGTH, // number of encoding name bytes including padding |
+ UCNVSEL_INDEX_SIZE = 15, // bytes following the DataHeader |
+ UCNVSEL_INDEX_COUNT = 16 |
+}; |
+ |
+/* |
+ * Serialized form of a UConverterSelector, formatVersion 1: |
+ * |
+ * The serialized form begins with a standard ICU DataHeader with a UDataInfo |
+ * as the template above. |
+ * This is followed by: |
+ * int32_t indexes[UCNVSEL_INDEX_COUNT]; // see index entry constants above |
+ * serialized UTrie2; // indexes[UCNVSEL_INDEX_TRIE_SIZE] bytes |
+ * uint32_t pv[indexes[UCNVSEL_INDEX_PV_COUNT]]; // bit vectors |
+ * char* encodingNames[indexes[UCNVSEL_INDEX_NAMES_LENGTH]]; // NUL-terminated strings + padding |
+ */ |
+ |
+/* serialize a selector */ |
+U_CAPI int32_t U_EXPORT2 |
+ucnvsel_serialize(const UConverterSelector* sel, |
+ void* buffer, int32_t bufferCapacity, UErrorCode* status) { |
+ // check if already failed |
+ if (U_FAILURE(*status)) { |
+ return 0; |
+ } |
+ // ensure args make sense! |
+ uint8_t *p = (uint8_t *)buffer; |
+ if (bufferCapacity < 0 || |
+ (bufferCapacity > 0 && (p == NULL || (U_POINTER_MASK_LSB(p, 3) != 0))) |
+ ) { |
+ *status = U_ILLEGAL_ARGUMENT_ERROR; |
+ return 0; |
+ } |
+ // add up the size of the serialized form |
+ int32_t serializedTrieSize = utrie2_serialize(sel->trie, NULL, 0, status); |
+ if (*status != U_BUFFER_OVERFLOW_ERROR && U_FAILURE(*status)) { |
+ return 0; |
+ } |
+ *status = U_ZERO_ERROR; |
+ |
+ DataHeader header; |
+ uprv_memset(&header, 0, sizeof(header)); |
+ header.dataHeader.headerSize = (uint16_t)((sizeof(header) + 15) & ~15); |
+ header.dataHeader.magic1 = 0xda; |
+ header.dataHeader.magic2 = 0x27; |
+ uprv_memcpy(&header.info, &dataInfo, sizeof(dataInfo)); |
+ |
+ int32_t indexes[UCNVSEL_INDEX_COUNT] = { |
+ serializedTrieSize, |
+ sel->pvCount, |
+ sel->encodingsCount, |
+ sel->encodingStrLength |
+ }; |
+ |
+ int32_t totalSize = |
+ header.dataHeader.headerSize + |
+ (int32_t)sizeof(indexes) + |
+ serializedTrieSize + |
+ sel->pvCount * 4 + |
+ sel->encodingStrLength; |
+ indexes[UCNVSEL_INDEX_SIZE] = totalSize - header.dataHeader.headerSize; |
+ if (totalSize > bufferCapacity) { |
+ *status = U_BUFFER_OVERFLOW_ERROR; |
+ return totalSize; |
+ } |
+ // ok, save! |
+ int32_t length = header.dataHeader.headerSize; |
+ uprv_memcpy(p, &header, sizeof(header)); |
+ uprv_memset(p + sizeof(header), 0, length - sizeof(header)); |
+ p += length; |
+ |
+ length = (int32_t)sizeof(indexes); |
+ uprv_memcpy(p, indexes, length); |
+ p += length; |
+ |
+ utrie2_serialize(sel->trie, p, serializedTrieSize, status); |
+ p += serializedTrieSize; |
+ |
+ length = sel->pvCount * 4; |
+ uprv_memcpy(p, sel->pv, length); |
+ p += length; |
+ |
+ uprv_memcpy(p, sel->encodings[0], sel->encodingStrLength); |
+ p += sel->encodingStrLength; |
+ |
+ return totalSize; |
+} |
+ |
+/** |
+ * swap a selector into the desired Endianness and Asciiness of |
+ * the system. Just as FYI, selectors are always saved in the format |
+ * of the system that created them. They are only converted if used |
+ * on another system. In other words, selectors created on different |
+ * system can be different even if the params are identical (endianness |
+ * and Asciiness differences only) |
+ * |
+ * @param ds pointer to data swapper containing swapping info |
+ * @param inData pointer to incoming data |
+ * @param length length of inData in bytes |
+ * @param outData pointer to output data. Capacity should |
+ * be at least equal to capacity of inData |
+ * @param status an in/out ICU UErrorCode |
+ * @return 0 on failure, number of bytes swapped on success |
+ * number of bytes swapped can be smaller than length |
+ */ |
+static int32_t |
+ucnvsel_swap(const UDataSwapper *ds, |
+ const void *inData, int32_t length, |
+ void *outData, UErrorCode *status) { |
+ /* udata_swapDataHeader checks the arguments */ |
+ int32_t headerSize = udata_swapDataHeader(ds, inData, length, outData, status); |
+ if(U_FAILURE(*status)) { |
+ return 0; |
+ } |
+ |
+ /* check data format and format version */ |
+ const UDataInfo *pInfo = (const UDataInfo *)((const char *)inData + 4); |
+ if(!( |
+ pInfo->dataFormat[0] == 0x43 && /* dataFormat="CSel" */ |
+ pInfo->dataFormat[1] == 0x53 && |
+ pInfo->dataFormat[2] == 0x65 && |
+ pInfo->dataFormat[3] == 0x6c |
+ )) { |
+ udata_printError(ds, "ucnvsel_swap(): data format %02x.%02x.%02x.%02x is not recognized as UConverterSelector data\n", |
+ pInfo->dataFormat[0], pInfo->dataFormat[1], |
+ pInfo->dataFormat[2], pInfo->dataFormat[3]); |
+ *status = U_INVALID_FORMAT_ERROR; |
+ return 0; |
+ } |
+ if(pInfo->formatVersion[0] != 1) { |
+ udata_printError(ds, "ucnvsel_swap(): format version %02x is not supported\n", |
+ pInfo->formatVersion[0]); |
+ *status = U_UNSUPPORTED_ERROR; |
+ return 0; |
+ } |
+ |
+ if(length >= 0) { |
+ length -= headerSize; |
+ if(length < 16*4) { |
+ udata_printError(ds, "ucnvsel_swap(): too few bytes (%d after header) for UConverterSelector data\n", |
+ length); |
+ *status = U_INDEX_OUTOFBOUNDS_ERROR; |
+ return 0; |
+ } |
+ } |
+ |
+ const uint8_t *inBytes = (const uint8_t *)inData + headerSize; |
+ uint8_t *outBytes = (uint8_t *)outData + headerSize; |
+ |
+ /* read the indexes */ |
+ const int32_t *inIndexes = (const int32_t *)inBytes; |
+ int32_t indexes[16]; |
+ int32_t i; |
+ for(i = 0; i < 16; ++i) { |
+ indexes[i] = udata_readInt32(ds, inIndexes[i]); |
+ } |
+ |
+ /* get the total length of the data */ |
+ int32_t size = indexes[UCNVSEL_INDEX_SIZE]; |
+ if(length >= 0) { |
+ if(length < size) { |
+ udata_printError(ds, "ucnvsel_swap(): too few bytes (%d after header) for all of UConverterSelector data\n", |
+ length); |
+ *status = U_INDEX_OUTOFBOUNDS_ERROR; |
+ return 0; |
+ } |
+ |
+ /* copy the data for inaccessible bytes */ |
+ if(inBytes != outBytes) { |
+ uprv_memcpy(outBytes, inBytes, size); |
+ } |
+ |
+ int32_t offset = 0, count; |
+ |
+ /* swap the int32_t indexes[] */ |
+ count = UCNVSEL_INDEX_COUNT*4; |
+ ds->swapArray32(ds, inBytes, count, outBytes, status); |
+ offset += count; |
+ |
+ /* swap the UTrie2 */ |
+ count = indexes[UCNVSEL_INDEX_TRIE_SIZE]; |
+ utrie2_swap(ds, inBytes + offset, count, outBytes + offset, status); |
+ offset += count; |
+ |
+ /* swap the uint32_t pv[] */ |
+ count = indexes[UCNVSEL_INDEX_PV_COUNT]*4; |
+ ds->swapArray32(ds, inBytes + offset, count, outBytes + offset, status); |
+ offset += count; |
+ |
+ /* swap the encoding names */ |
+ count = indexes[UCNVSEL_INDEX_NAMES_LENGTH]; |
+ ds->swapInvChars(ds, inBytes + offset, count, outBytes + offset, status); |
+ offset += count; |
+ |
+ U_ASSERT(offset == size); |
+ } |
+ |
+ return headerSize + size; |
+} |
+ |
+/* unserialize a selector */ |
+U_CAPI UConverterSelector* U_EXPORT2 |
+ucnvsel_openFromSerialized(const void* buffer, int32_t length, UErrorCode* status) { |
+ // check if already failed |
+ if (U_FAILURE(*status)) { |
+ return NULL; |
+ } |
+ // ensure args make sense! |
+ const uint8_t *p = (const uint8_t *)buffer; |
+ if (length <= 0 || |
+ (length > 0 && (p == NULL || (U_POINTER_MASK_LSB(p, 3) != 0))) |
+ ) { |
+ *status = U_ILLEGAL_ARGUMENT_ERROR; |
+ return NULL; |
+ } |
+ // header |
+ if (length < 32) { |
+ // not even enough space for a minimal header |
+ *status = U_INDEX_OUTOFBOUNDS_ERROR; |
+ return NULL; |
+ } |
+ const DataHeader *pHeader = (const DataHeader *)p; |
+ if (!( |
+ pHeader->dataHeader.magic1==0xda && |
+ pHeader->dataHeader.magic2==0x27 && |
+ pHeader->info.dataFormat[0] == 0x43 && |
+ pHeader->info.dataFormat[1] == 0x53 && |
+ pHeader->info.dataFormat[2] == 0x65 && |
+ pHeader->info.dataFormat[3] == 0x6c |
+ )) { |
+ /* header not valid or dataFormat not recognized */ |
+ *status = U_INVALID_FORMAT_ERROR; |
+ return NULL; |
+ } |
+ if (pHeader->info.formatVersion[0] != 1) { |
+ *status = U_UNSUPPORTED_ERROR; |
+ return NULL; |
+ } |
+ uint8_t* swapped = NULL; |
+ if (pHeader->info.isBigEndian != U_IS_BIG_ENDIAN || |
+ pHeader->info.charsetFamily != U_CHARSET_FAMILY |
+ ) { |
+ // swap the data |
+ UDataSwapper *ds = |
+ udata_openSwapperForInputData(p, length, U_IS_BIG_ENDIAN, U_CHARSET_FAMILY, status); |
+ int32_t totalSize = ucnvsel_swap(ds, p, -1, NULL, status); |
+ if (U_FAILURE(*status)) { |
+ udata_closeSwapper(ds); |
+ return NULL; |
+ } |
+ if (length < totalSize) { |
+ udata_closeSwapper(ds); |
+ *status = U_INDEX_OUTOFBOUNDS_ERROR; |
+ return NULL; |
+ } |
+ swapped = (uint8_t*)uprv_malloc(totalSize); |
+ if (swapped == NULL) { |
+ udata_closeSwapper(ds); |
+ *status = U_MEMORY_ALLOCATION_ERROR; |
+ return NULL; |
+ } |
+ ucnvsel_swap(ds, p, length, swapped, status); |
+ udata_closeSwapper(ds); |
+ if (U_FAILURE(*status)) { |
+ uprv_free(swapped); |
+ return NULL; |
+ } |
+ p = swapped; |
+ pHeader = (const DataHeader *)p; |
+ } |
+ if (length < (pHeader->dataHeader.headerSize + 16 * 4)) { |
+ // not even enough space for the header and the indexes |
+ uprv_free(swapped); |
+ *status = U_INDEX_OUTOFBOUNDS_ERROR; |
+ return NULL; |
+ } |
+ p += pHeader->dataHeader.headerSize; |
+ length -= pHeader->dataHeader.headerSize; |
+ // indexes |
+ const int32_t *indexes = (const int32_t *)p; |
+ if (length < indexes[UCNVSEL_INDEX_SIZE]) { |
+ uprv_free(swapped); |
+ *status = U_INDEX_OUTOFBOUNDS_ERROR; |
+ return NULL; |
+ } |
+ p += UCNVSEL_INDEX_COUNT * 4; |
+ // create and populate the selector object |
+ UConverterSelector* sel = (UConverterSelector*)uprv_malloc(sizeof(UConverterSelector)); |
+ char **encodings = |
+ (char **)uprv_malloc( |
+ indexes[UCNVSEL_INDEX_NAMES_COUNT] * sizeof(char *)); |
+ if (sel == NULL || encodings == NULL) { |
+ uprv_free(swapped); |
+ uprv_free(sel); |
+ uprv_free(encodings); |
+ *status = U_MEMORY_ALLOCATION_ERROR; |
+ return NULL; |
+ } |
+ uprv_memset(sel, 0, sizeof(UConverterSelector)); |
+ sel->pvCount = indexes[UCNVSEL_INDEX_PV_COUNT]; |
+ sel->encodings = encodings; |
+ sel->encodingsCount = indexes[UCNVSEL_INDEX_NAMES_COUNT]; |
+ sel->encodingStrLength = indexes[UCNVSEL_INDEX_NAMES_LENGTH]; |
+ sel->swapped = swapped; |
+ // trie |
+ sel->trie = utrie2_openFromSerialized(UTRIE2_16_VALUE_BITS, |
+ p, indexes[UCNVSEL_INDEX_TRIE_SIZE], NULL, |
+ status); |
+ p += indexes[UCNVSEL_INDEX_TRIE_SIZE]; |
+ if (U_FAILURE(*status)) { |
+ ucnvsel_close(sel); |
+ return NULL; |
+ } |
+ // bit vectors |
+ sel->pv = (uint32_t *)p; |
+ p += sel->pvCount * 4; |
+ // encoding names |
+ char* s = (char*)p; |
+ for (int32_t i = 0; i < sel->encodingsCount; ++i) { |
+ sel->encodings[i] = s; |
+ s += uprv_strlen(s) + 1; |
+ } |
+ p += sel->encodingStrLength; |
+ |
+ return sel; |
+} |
+ |
+// a bunch of functions for the enumeration thingie! Nothing fancy here. Just |
+// iterate over the selected encodings |
+struct Enumerator { |
+ int16_t* index; |
+ int16_t length; |
+ int16_t cur; |
+ const UConverterSelector* sel; |
+}; |
+ |
+U_CDECL_BEGIN |
+ |
+static void U_CALLCONV |
+ucnvsel_close_selector_iterator(UEnumeration *enumerator) { |
+ uprv_free(((Enumerator*)(enumerator->context))->index); |
+ uprv_free(enumerator->context); |
+ uprv_free(enumerator); |
+} |
+ |
+ |
+static int32_t U_CALLCONV |
+ucnvsel_count_encodings(UEnumeration *enumerator, UErrorCode *status) { |
+ // check if already failed |
+ if (U_FAILURE(*status)) { |
+ return 0; |
+ } |
+ return ((Enumerator*)(enumerator->context))->length; |
+} |
+ |
+ |
+static const char* U_CALLCONV ucnvsel_next_encoding(UEnumeration* enumerator, |
+ int32_t* resultLength, |
+ UErrorCode* status) { |
+ // check if already failed |
+ if (U_FAILURE(*status)) { |
+ return NULL; |
+ } |
+ |
+ int16_t cur = ((Enumerator*)(enumerator->context))->cur; |
+ const UConverterSelector* sel; |
+ const char* result; |
+ if (cur >= ((Enumerator*)(enumerator->context))->length) { |
+ return NULL; |
+ } |
+ sel = ((Enumerator*)(enumerator->context))->sel; |
+ result = sel->encodings[((Enumerator*)(enumerator->context))->index[cur] ]; |
+ ((Enumerator*)(enumerator->context))->cur++; |
+ if (resultLength) { |
+ *resultLength = (int32_t)uprv_strlen(result); |
+ } |
+ return result; |
+} |
+ |
+static void U_CALLCONV ucnvsel_reset_iterator(UEnumeration* enumerator, |
+ UErrorCode* status) { |
+ // check if already failed |
+ if (U_FAILURE(*status)) { |
+ return ; |
+ } |
+ ((Enumerator*)(enumerator->context))->cur = 0; |
+} |
+ |
+U_CDECL_END |
+ |
+ |
+static const UEnumeration defaultEncodings = { |
+ NULL, |
+ NULL, |
+ ucnvsel_close_selector_iterator, |
+ ucnvsel_count_encodings, |
+ uenum_unextDefault, |
+ ucnvsel_next_encoding, |
+ ucnvsel_reset_iterator |
+}; |
+ |
+ |
+// internal fn to intersect two sets of masks |
+// returns whether the mask has reduced to all zeros |
+static UBool intersectMasks(uint32_t* dest, const uint32_t* source1, int32_t len) { |
+ int32_t i; |
+ uint32_t oredDest = 0; |
+ for (i = 0 ; i < len ; ++i) { |
+ oredDest |= (dest[i] &= source1[i]); |
+ } |
+ return oredDest == 0; |
+} |
+ |
+// internal fn to count how many 1's are there in a mask |
+// algorithm taken from http://graphics.stanford.edu/~seander/bithacks.html |
+static int16_t countOnes(uint32_t* mask, int32_t len) { |
+ int32_t i, totalOnes = 0; |
+ for (i = 0 ; i < len ; ++i) { |
+ uint32_t ent = mask[i]; |
+ for (; ent; totalOnes++) |
+ { |
+ ent &= ent - 1; // clear the least significant bit set |
+ } |
+ } |
+ return totalOnes; |
+} |
+ |
+ |
+/* internal function! */ |
+static UEnumeration *selectForMask(const UConverterSelector* sel, |
+ uint32_t *mask, UErrorCode *status) { |
+ // this is the context we will use. Store a table of indices to which |
+ // encodings are legit. |
+ struct Enumerator* result = (Enumerator*)uprv_malloc(sizeof(Enumerator)); |
+ if (result == NULL) { |
+ uprv_free(mask); |
+ *status = U_MEMORY_ALLOCATION_ERROR; |
+ return NULL; |
+ } |
+ result->index = NULL; // this will be allocated later! |
+ result->length = result->cur = 0; |
+ result->sel = sel; |
+ |
+ UEnumeration *en = (UEnumeration *)uprv_malloc(sizeof(UEnumeration)); |
+ if (en == NULL) { |
+ // TODO(markus): Combine Enumerator and UEnumeration into one struct. |
+ uprv_free(mask); |
+ uprv_free(result); |
+ *status = U_MEMORY_ALLOCATION_ERROR; |
+ return NULL; |
+ } |
+ memcpy(en, &defaultEncodings, sizeof(UEnumeration)); |
+ en->context = result; |
+ |
+ int32_t columns = (sel->encodingsCount+31)/32; |
+ int16_t numOnes = countOnes(mask, columns); |
+ // now, we know the exact space we need for index |
+ if (numOnes > 0) { |
+ result->index = (int16_t*) uprv_malloc(numOnes * sizeof(int16_t)); |
+ |
+ int32_t i, j; |
+ int16_t k = 0; |
+ for (j = 0 ; j < columns; j++) { |
+ uint32_t v = mask[j]; |
+ for (i = 0 ; i < 32 && k < sel->encodingsCount; i++, k++) { |
+ if ((v & 1) != 0) { |
+ result->index[result->length++] = k; |
+ } |
+ v >>= 1; |
+ } |
+ } |
+ } //otherwise, index will remain NULL (and will never be touched by |
+ //the enumerator code anyway) |
+ uprv_free(mask); |
+ return en; |
+} |
+ |
+/* check a string against the selector - UTF16 version */ |
+U_CAPI UEnumeration * U_EXPORT2 |
+ucnvsel_selectForString(const UConverterSelector* sel, |
+ const UChar *s, int32_t length, UErrorCode *status) { |
+ // check if already failed |
+ if (U_FAILURE(*status)) { |
+ return NULL; |
+ } |
+ // ensure args make sense! |
+ if (sel == NULL || (s == NULL && length != 0)) { |
+ *status = U_ILLEGAL_ARGUMENT_ERROR; |
+ return NULL; |
+ } |
+ |
+ int32_t columns = (sel->encodingsCount+31)/32; |
+ uint32_t* mask = (uint32_t*) uprv_malloc(columns * 4); |
+ if (mask == NULL) { |
+ *status = U_MEMORY_ALLOCATION_ERROR; |
+ return NULL; |
+ } |
+ uprv_memset(mask, ~0, columns *4); |
+ |
+ const UChar *limit; |
+ if (length >= 0) { |
+ limit = s + length; |
+ } else { |
+ limit = NULL; |
+ } |
+ |
+ while (limit == NULL ? *s != 0 : s != limit) { |
+ UChar32 c; |
+ uint16_t pvIndex; |
+ UTRIE2_U16_NEXT16(sel->trie, s, limit, c, pvIndex); |
+ if (intersectMasks(mask, sel->pv+pvIndex, columns)) { |
+ break; |
+ } |
+ } |
+ return selectForMask(sel, mask, status); |
+} |
+ |
+/* check a string against the selector - UTF8 version */ |
+U_CAPI UEnumeration * U_EXPORT2 |
+ucnvsel_selectForUTF8(const UConverterSelector* sel, |
+ const char *s, int32_t length, UErrorCode *status) { |
+ // check if already failed |
+ if (U_FAILURE(*status)) { |
+ return NULL; |
+ } |
+ // ensure args make sense! |
+ if (sel == NULL || (s == NULL && length != 0)) { |
+ *status = U_ILLEGAL_ARGUMENT_ERROR; |
+ return NULL; |
+ } |
+ |
+ int32_t columns = (sel->encodingsCount+31)/32; |
+ uint32_t* mask = (uint32_t*) uprv_malloc(columns * 4); |
+ if (mask == NULL) { |
+ *status = U_MEMORY_ALLOCATION_ERROR; |
+ return NULL; |
+ } |
+ uprv_memset(mask, ~0, columns *4); |
+ |
+ if (length < 0) { |
+ length = (int32_t)uprv_strlen(s); |
+ } |
+ const char *limit = s + length; |
+ |
+ while (s != limit) { |
+ uint16_t pvIndex; |
+ UTRIE2_U8_NEXT16(sel->trie, s, limit, pvIndex); |
+ if (intersectMasks(mask, sel->pv+pvIndex, columns)) { |
+ break; |
+ } |
+ } |
+ return selectForMask(sel, mask, status); |
+} |
Property changes on: icu46/source/common/ucnvsel.cpp |
___________________________________________________________________ |
Added: svn:eol-style |
+ LF |