| OLD | NEW |
|---|
| (Empty) | |
| 1 /* |
| 2 ******************************************************************************* |
| 3 * |
| 4 * Copyright (C) 2003-2007, International Business Machines |
| 5 * Corporation and others. All Rights Reserved. |
| 6 * |
| 7 ******************************************************************************* |
| 8 * file name: gencnvex.c |
| 9 * encoding: US-ASCII |
| 10 * tab size: 8 (not used) |
| 11 * indentation:4 |
| 12 * |
| 13 * created on: 2003oct12 |
| 14 * created by: Markus W. Scherer |
| 15 */ |
| 16 |
| 17 #include <stdio.h> |
| 18 #include "unicode/utypes.h" |
| 19 #include "unicode/ustring.h" |
| 20 #include "cstring.h" |
| 21 #include "cmemory.h" |
| 22 #include "ucnv_cnv.h" |
| 23 #include "ucnvmbcs.h" |
| 24 #include "toolutil.h" |
| 25 #include "unewdata.h" |
| 26 #include "ucm.h" |
| 27 #include "makeconv.h" |
| 28 #include "genmbcs.h" |
| 29 |
| 30 #define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0])) |
| 31 |
| 32 |
| 33 static void |
| 34 CnvExtClose(NewConverter *cnvData); |
| 35 |
| 36 static UBool |
| 37 CnvExtIsValid(NewConverter *cnvData, |
| 38 const uint8_t *bytes, int32_t length); |
| 39 |
| 40 static UBool |
| 41 CnvExtAddTable(NewConverter *cnvData, UCMTable *table, UConverterStaticData *sta
ticData); |
| 42 |
| 43 static uint32_t |
| 44 CnvExtWrite(NewConverter *cnvData, const UConverterStaticData *staticData, |
| 45 UNewDataMemory *pData, int32_t tableType); |
| 46 |
| 47 typedef struct CnvExtData { |
| 48 NewConverter newConverter; |
| 49 |
| 50 UCMFile *ucm; |
| 51 |
| 52 /* toUnicode (state table in ucm->states) */ |
| 53 UToolMemory *toUTable, *toUUChars; |
| 54 |
| 55 /* fromUnicode */ |
| 56 UToolMemory *fromUTableUChars, *fromUTableValues, *fromUBytes; |
| 57 |
| 58 uint16_t stage1[MBCS_STAGE_1_SIZE]; |
| 59 uint16_t stage2[MBCS_STAGE_2_SIZE]; |
| 60 uint16_t stage3[0x10000<<UCNV_EXT_STAGE_2_LEFT_SHIFT]; /* 0x10000 because of
16-bit stage 2/3 indexes */ |
| 61 uint32_t stage3b[0x10000]; |
| 62 |
| 63 int32_t stage1Top, stage2Top, stage3Top, stage3bTop; |
| 64 |
| 65 /* for stage3 compaction of <subchar1> |2 mappings */ |
| 66 uint16_t stage3Sub1Block; |
| 67 |
| 68 /* statistics */ |
| 69 int32_t |
| 70 maxInBytes, maxOutBytes, maxBytesPerUChar, |
| 71 maxInUChars, maxOutUChars, maxUCharsPerByte; |
| 72 } CnvExtData; |
| 73 |
| 74 NewConverter * |
| 75 CnvExtOpen(UCMFile *ucm) { |
| 76 CnvExtData *extData; |
| 77 |
| 78 extData=(CnvExtData *)uprv_malloc(sizeof(CnvExtData)); |
| 79 if(extData==NULL) { |
| 80 printf("out of memory\n"); |
| 81 exit(U_MEMORY_ALLOCATION_ERROR); |
| 82 } |
| 83 uprv_memset(extData, 0, sizeof(CnvExtData)); |
| 84 |
| 85 extData->ucm=ucm; /* aliased, not owned */ |
| 86 |
| 87 extData->newConverter.close=CnvExtClose; |
| 88 extData->newConverter.isValid=CnvExtIsValid; |
| 89 extData->newConverter.addTable=CnvExtAddTable; |
| 90 extData->newConverter.write=CnvExtWrite; |
| 91 return &extData->newConverter; |
| 92 } |
| 93 |
| 94 static void |
| 95 CnvExtClose(NewConverter *cnvData) { |
| 96 CnvExtData *extData=(CnvExtData *)cnvData; |
| 97 if(extData!=NULL) { |
| 98 utm_close(extData->toUTable); |
| 99 utm_close(extData->toUUChars); |
| 100 utm_close(extData->fromUTableUChars); |
| 101 utm_close(extData->fromUTableValues); |
| 102 utm_close(extData->fromUBytes); |
| 103 } |
| 104 } |
| 105 |
| 106 /* we do not expect this to be called */ |
| 107 static UBool |
| 108 CnvExtIsValid(NewConverter *cnvData, |
| 109 const uint8_t *bytes, int32_t length) { |
| 110 return FALSE; |
| 111 } |
| 112 |
| 113 static uint32_t |
| 114 CnvExtWrite(NewConverter *cnvData, const UConverterStaticData *staticData, |
| 115 UNewDataMemory *pData, int32_t tableType) { |
| 116 CnvExtData *extData=(CnvExtData *)cnvData; |
| 117 int32_t length, top, headerSize; |
| 118 |
| 119 int32_t indexes[UCNV_EXT_INDEXES_MIN_LENGTH]={ 0 }; |
| 120 |
| 121 if(tableType&TABLE_BASE) { |
| 122 headerSize=0; |
| 123 } else { |
| 124 _MBCSHeader header={ { 0, 0, 0, 0 }, 0, 0, 0, 0, 0, 0, 0 }; |
| 125 |
| 126 /* write the header and base table name for an extension-only table */ |
| 127 length=(int32_t)uprv_strlen(extData->ucm->baseName)+1; |
| 128 while(length&3) { |
| 129 /* add padding */ |
| 130 extData->ucm->baseName[length++]=0; |
| 131 } |
| 132 |
| 133 headerSize=MBCS_HEADER_V4_LENGTH*4+length; |
| 134 |
| 135 /* fill the header */ |
| 136 header.version[0]=4; |
| 137 header.version[1]=2; |
| 138 header.flags=(uint32_t)((headerSize<<8)|MBCS_OUTPUT_EXT_ONLY); |
| 139 |
| 140 /* write the header and the base table name */ |
| 141 udata_writeBlock(pData, &header, MBCS_HEADER_V4_LENGTH*4); |
| 142 udata_writeBlock(pData, extData->ucm->baseName, length); |
| 143 } |
| 144 |
| 145 /* fill indexes[] - offsets/indexes are in units of the target array */ |
| 146 top=0; |
| 147 |
| 148 indexes[UCNV_EXT_INDEXES_LENGTH]=length=UCNV_EXT_INDEXES_MIN_LENGTH; |
| 149 top+=length*4; |
| 150 |
| 151 indexes[UCNV_EXT_TO_U_INDEX]=top; |
| 152 indexes[UCNV_EXT_TO_U_LENGTH]=length=utm_countItems(extData->toUTable); |
| 153 top+=length*4; |
| 154 |
| 155 indexes[UCNV_EXT_TO_U_UCHARS_INDEX]=top; |
| 156 indexes[UCNV_EXT_TO_U_UCHARS_LENGTH]=length=utm_countItems(extData->toUUChar
s); |
| 157 top+=length*2; |
| 158 |
| 159 indexes[UCNV_EXT_FROM_U_UCHARS_INDEX]=top; |
| 160 length=utm_countItems(extData->fromUTableUChars); |
| 161 top+=length*2; |
| 162 |
| 163 if(top&3) { |
| 164 /* add padding */ |
| 165 *((UChar *)utm_alloc(extData->fromUTableUChars))=0; |
| 166 *((uint32_t *)utm_alloc(extData->fromUTableValues))=0; |
| 167 ++length; |
| 168 top+=2; |
| 169 } |
| 170 indexes[UCNV_EXT_FROM_U_LENGTH]=length; |
| 171 |
| 172 indexes[UCNV_EXT_FROM_U_VALUES_INDEX]=top; |
| 173 top+=length*4; |
| 174 |
| 175 indexes[UCNV_EXT_FROM_U_BYTES_INDEX]=top; |
| 176 length=utm_countItems(extData->fromUBytes); |
| 177 top+=length; |
| 178 |
| 179 if(top&1) { |
| 180 /* add padding */ |
| 181 *((uint8_t *)utm_alloc(extData->fromUBytes))=0; |
| 182 ++length; |
| 183 ++top; |
| 184 } |
| 185 indexes[UCNV_EXT_FROM_U_BYTES_LENGTH]=length; |
| 186 |
| 187 indexes[UCNV_EXT_FROM_U_STAGE_12_INDEX]=top; |
| 188 indexes[UCNV_EXT_FROM_U_STAGE_1_LENGTH]=length=extData->stage1Top; |
| 189 indexes[UCNV_EXT_FROM_U_STAGE_12_LENGTH]=length+=extData->stage2Top; |
| 190 top+=length*2; |
| 191 |
| 192 indexes[UCNV_EXT_FROM_U_STAGE_3_INDEX]=top; |
| 193 length=extData->stage3Top; |
| 194 top+=length*2; |
| 195 |
| 196 if(top&3) { |
| 197 /* add padding */ |
| 198 extData->stage3[extData->stage3Top++]=0; |
| 199 ++length; |
| 200 top+=2; |
| 201 } |
| 202 indexes[UCNV_EXT_FROM_U_STAGE_3_LENGTH]=length; |
| 203 |
| 204 indexes[UCNV_EXT_FROM_U_STAGE_3B_INDEX]=top; |
| 205 indexes[UCNV_EXT_FROM_U_STAGE_3B_LENGTH]=length=extData->stage3bTop; |
| 206 top+=length*4; |
| 207 |
| 208 indexes[UCNV_EXT_SIZE]=top; |
| 209 |
| 210 /* statistics */ |
| 211 indexes[UCNV_EXT_COUNT_BYTES]= |
| 212 (extData->maxInBytes<<16)| |
| 213 (extData->maxOutBytes<<8)| |
| 214 extData->maxBytesPerUChar; |
| 215 indexes[UCNV_EXT_COUNT_UCHARS]= |
| 216 (extData->maxInUChars<<16)| |
| 217 (extData->maxOutUChars<<8)| |
| 218 extData->maxUCharsPerByte; |
| 219 |
| 220 indexes[UCNV_EXT_FLAGS]=extData->ucm->ext->unicodeMask; |
| 221 |
| 222 /* write the extension data */ |
| 223 udata_writeBlock(pData, indexes, sizeof(indexes)); |
| 224 udata_writeBlock(pData, utm_getStart(extData->toUTable), indexes[UCNV_EXT_TO
_U_LENGTH]*4); |
| 225 udata_writeBlock(pData, utm_getStart(extData->toUUChars), indexes[UCNV_EXT_T
O_U_UCHARS_LENGTH]*2); |
| 226 |
| 227 udata_writeBlock(pData, utm_getStart(extData->fromUTableUChars), indexes[UCN
V_EXT_FROM_U_LENGTH]*2); |
| 228 udata_writeBlock(pData, utm_getStart(extData->fromUTableValues), indexes[UCN
V_EXT_FROM_U_LENGTH]*4); |
| 229 udata_writeBlock(pData, utm_getStart(extData->fromUBytes), indexes[UCNV_EXT_
FROM_U_BYTES_LENGTH]); |
| 230 |
| 231 udata_writeBlock(pData, extData->stage1, extData->stage1Top*2); |
| 232 udata_writeBlock(pData, extData->stage2, extData->stage2Top*2); |
| 233 udata_writeBlock(pData, extData->stage3, extData->stage3Top*2); |
| 234 udata_writeBlock(pData, extData->stage3b, extData->stage3bTop*4); |
| 235 |
| 236 #if 0 |
| 237 { |
| 238 int32_t i, j; |
| 239 |
| 240 length=extData->stage1Top; |
| 241 printf("\nstage1[%x]:\n", length); |
| 242 |
| 243 for(i=0; i<length; ++i) { |
| 244 if(extData->stage1[i]!=length) { |
| 245 printf("stage1[%04x]=%04x\n", i, extData->stage1[i]); |
| 246 } |
| 247 } |
| 248 |
| 249 j=length; |
| 250 length=extData->stage2Top; |
| 251 printf("\nstage2[%x]:\n", length); |
| 252 |
| 253 for(i=0; i<length; ++j, ++i) { |
| 254 if(extData->stage2[i]!=0) { |
| 255 printf("stage12[%04x]=%04x\n", j, extData->stage2[i]); |
| 256 } |
| 257 } |
| 258 |
| 259 length=extData->stage3Top; |
| 260 printf("\nstage3[%x]:\n", length); |
| 261 |
| 262 for(i=0; i<length; ++i) { |
| 263 if(extData->stage3[i]!=0) { |
| 264 printf("stage3[%04x]=%04x\n", i, extData->stage3[i]); |
| 265 } |
| 266 } |
| 267 |
| 268 length=extData->stage3bTop; |
| 269 printf("\nstage3b[%x]:\n", length); |
| 270 |
| 271 for(i=0; i<length; ++i) { |
| 272 if(extData->stage3b[i]!=0) { |
| 273 printf("stage3b[%04x]=%08x\n", i, extData->stage3b[i]); |
| 274 } |
| 275 } |
| 276 } |
| 277 #endif |
| 278 |
| 279 if(VERBOSE) { |
| 280 printf("size of extension data: %ld\n", (long)top); |
| 281 } |
| 282 |
| 283 /* return the number of bytes that should have been written */ |
| 284 return (uint32_t)(headerSize+top); |
| 285 } |
| 286 |
| 287 /* to Unicode --------------------------------------------------------------- */ |
| 288 |
| 289 /* |
| 290 * Remove fromUnicode fallbacks and SUB mappings which are irrelevant for |
| 291 * the toUnicode table. |
| 292 * This includes mappings with MBCS_FROM_U_EXT_FLAG which were suitable |
| 293 * for the base toUnicode table but not for the base fromUnicode table. |
| 294 * The table must be sorted. |
| 295 * Modifies previous data in the reverseMap. |
| 296 */ |
| 297 static int32_t |
| 298 reduceToUMappings(UCMTable *table) { |
| 299 UCMapping *mappings; |
| 300 int32_t *map; |
| 301 int32_t i, j, count; |
| 302 int8_t flag; |
| 303 |
| 304 mappings=table->mappings; |
| 305 map=table->reverseMap; |
| 306 count=table->mappingsLength; |
| 307 |
| 308 /* leave the map alone for the initial mappings with desired flags */ |
| 309 for(i=j=0; i<count; ++i) { |
| 310 flag=mappings[map[i]].f; |
| 311 if(flag!=0 && flag!=3) { |
| 312 break; |
| 313 } |
| 314 } |
| 315 |
| 316 /* reduce from here to the rest */ |
| 317 for(j=i; i<count; ++i) { |
| 318 flag=mappings[map[i]].f; |
| 319 if(flag==0 || flag==3) { |
| 320 map[j++]=map[i]; |
| 321 } |
| 322 } |
| 323 |
| 324 return j; |
| 325 } |
| 326 |
| 327 static uint32_t |
| 328 getToUnicodeValue(CnvExtData *extData, UCMTable *table, UCMapping *m) { |
| 329 UChar32 *u32; |
| 330 UChar *u; |
| 331 uint32_t value; |
| 332 int32_t u16Length, ratio; |
| 333 UErrorCode errorCode; |
| 334 |
| 335 /* write the Unicode result code point or string index */ |
| 336 if(m->uLen==1) { |
| 337 u16Length=U16_LENGTH(m->u); |
| 338 value=(uint32_t)(UCNV_EXT_TO_U_MIN_CODE_POINT+m->u); |
| 339 } else { |
| 340 /* the parser enforces m->uLen<=UCNV_EXT_MAX_UCHARS */ |
| 341 |
| 342 /* get the result code point string and its 16-bit string length */ |
| 343 u32=UCM_GET_CODE_POINTS(table, m); |
| 344 errorCode=U_ZERO_ERROR; |
| 345 u_strFromUTF32(NULL, 0, &u16Length, u32, m->uLen, &errorCode); |
| 346 if(U_FAILURE(errorCode) && errorCode!=U_BUFFER_OVERFLOW_ERROR) { |
| 347 exit(errorCode); |
| 348 } |
| 349 |
| 350 /* allocate it and put its length and index into the value */ |
| 351 value= |
| 352 (((uint32_t)m->uLen+UCNV_EXT_TO_U_LENGTH_OFFSET)<<UCNV_EXT_TO_U_LENG
TH_SHIFT)| |
| 353 ((uint32_t)utm_countItems(extData->toUUChars)); |
| 354 u=utm_allocN(extData->toUUChars, u16Length); |
| 355 |
| 356 /* write the result 16-bit string */ |
| 357 errorCode=U_ZERO_ERROR; |
| 358 u_strFromUTF32(u, u16Length, NULL, u32, m->uLen, &errorCode); |
| 359 if(U_FAILURE(errorCode) && errorCode!=U_BUFFER_OVERFLOW_ERROR) { |
| 360 exit(errorCode); |
| 361 } |
| 362 } |
| 363 if(m->f==0) { |
| 364 value|=UCNV_EXT_TO_U_ROUNDTRIP_FLAG; |
| 365 } |
| 366 |
| 367 /* update statistics */ |
| 368 if(m->bLen>extData->maxInBytes) { |
| 369 extData->maxInBytes=m->bLen; |
| 370 } |
| 371 if(u16Length>extData->maxOutUChars) { |
| 372 extData->maxOutUChars=u16Length; |
| 373 } |
| 374 |
| 375 ratio=(u16Length+(m->bLen-1))/m->bLen; |
| 376 if(ratio>extData->maxUCharsPerByte) { |
| 377 extData->maxUCharsPerByte=ratio; |
| 378 } |
| 379 |
| 380 return value; |
| 381 } |
| 382 |
| 383 /* |
| 384 * Recursive toUTable generator core function. |
| 385 * Preconditions: |
| 386 * - start<limit (There is at least one mapping.) |
| 387 * - The mappings are sorted lexically. (Access is through the reverseMap.) |
| 388 * - All mappings between start and limit have input sequences that share |
| 389 * the same prefix of unitIndex length, and therefore all of these sequences |
| 390 * are at least unitIndex+1 long. |
| 391 * - There are only relevant mappings available through the reverseMap, |
| 392 * see reduceToUMappings(). |
| 393 * |
| 394 * One function invocation generates one section table. |
| 395 * |
| 396 * Steps: |
| 397 * 1. Count the number of unique unit values and get the low/high unit values |
| 398 * that occur at unitIndex. |
| 399 * 2. Allocate the section table with possible optimization for linear access. |
| 400 * 3. Write temporary version of the section table with start indexes of |
| 401 * subsections, each corresponding to one unit value at unitIndex. |
| 402 * 4. Iterate through the table once more, and depending on the subsection lengt
h: |
| 403 * 0: write 0 as a result value (unused byte in linear-access section table) |
| 404 * >0: if there is one mapping with an input unit sequence of unitIndex+1 |
| 405 * then defaultValue=compute the mapping result for this whole sequence |
| 406 * else defaultValue=0 |
| 407 * |
| 408 * recurse into the subsection |
| 409 */ |
| 410 static UBool |
| 411 generateToUTable(CnvExtData *extData, UCMTable *table, |
| 412 int32_t start, int32_t limit, int32_t unitIndex, |
| 413 uint32_t defaultValue) { |
| 414 UCMapping *mappings, *m; |
| 415 int32_t *map; |
| 416 int32_t i, j, uniqueCount, count, subStart, subLimit; |
| 417 |
| 418 uint8_t *bytes; |
| 419 int32_t low, high, prev; |
| 420 |
| 421 uint32_t *section; |
| 422 |
| 423 mappings=table->mappings; |
| 424 map=table->reverseMap; |
| 425 |
| 426 /* step 1: examine the input units; set low, high, uniqueCount */ |
| 427 m=mappings+map[start]; |
| 428 bytes=UCM_GET_BYTES(table, m); |
| 429 low=bytes[unitIndex]; |
| 430 uniqueCount=1; |
| 431 |
| 432 prev=high=low; |
| 433 for(i=start+1; i<limit; ++i) { |
| 434 m=mappings+map[i]; |
| 435 bytes=UCM_GET_BYTES(table, m); |
| 436 high=bytes[unitIndex]; |
| 437 |
| 438 if(high!=prev) { |
| 439 prev=high; |
| 440 ++uniqueCount; |
| 441 } |
| 442 } |
| 443 |
| 444 /* step 2: allocate the section; set count, section */ |
| 445 count=(high-low)+1; |
| 446 if(count<0x100 && (unitIndex==0 || uniqueCount>=(3*count)/4)) { |
| 447 /* |
| 448 * for the root table and for fairly full tables: |
| 449 * allocate for direct, linear array access |
| 450 * by keeping count, to write an entry for each unit value |
| 451 * from low to high |
| 452 * exception: use a compact table if count==0x100 because |
| 453 * that cannot be encoded in the length byte |
| 454 */ |
| 455 } else { |
| 456 count=uniqueCount; |
| 457 } |
| 458 |
| 459 if(count>=0x100) { |
| 460 fprintf(stderr, "error: toUnicode extension table section overflow: %ld
section entries\n", (long)count); |
| 461 return FALSE; |
| 462 } |
| 463 |
| 464 /* allocate the section: 1 entry for the header + count for the items */ |
| 465 section=(uint32_t *)utm_allocN(extData->toUTable, 1+count); |
| 466 |
| 467 /* write the section header */ |
| 468 *section++=((uint32_t)count<<UCNV_EXT_TO_U_BYTE_SHIFT)|defaultValue; |
| 469 |
| 470 /* step 3: write temporary section table with subsection starts */ |
| 471 prev=low-1; /* just before low to prevent empty subsections before low */ |
| 472 j=0; /* section table index */ |
| 473 for(i=start; i<limit; ++i) { |
| 474 m=mappings+map[i]; |
| 475 bytes=UCM_GET_BYTES(table, m); |
| 476 high=bytes[unitIndex]; |
| 477 |
| 478 if(high!=prev) { |
| 479 /* start of a new subsection for unit high */ |
| 480 if(count>uniqueCount) { |
| 481 /* write empty subsections for unused units in a linear table */ |
| 482 while(++prev<high) { |
| 483 section[j++]=((uint32_t)prev<<UCNV_EXT_TO_U_BYTE_SHIFT)|(uin
t32_t)i; |
| 484 } |
| 485 } else { |
| 486 prev=high; |
| 487 } |
| 488 |
| 489 /* write the entry with the subsection start */ |
| 490 section[j++]=((uint32_t)high<<UCNV_EXT_TO_U_BYTE_SHIFT)|(uint32_t)i; |
| 491 } |
| 492 } |
| 493 /* assert(j==count) */ |
| 494 |
| 495 /* step 4: recurse and write results */ |
| 496 subLimit=UCNV_EXT_TO_U_GET_VALUE(section[0]); |
| 497 for(j=0; j<count; ++j) { |
| 498 subStart=subLimit; |
| 499 subLimit= (j+1)<count ? UCNV_EXT_TO_U_GET_VALUE(section[j+1]) : limit; |
| 500 |
| 501 /* remove the subStart temporary value */ |
| 502 section[j]&=~UCNV_EXT_TO_U_VALUE_MASK; |
| 503 |
| 504 if(subStart==subLimit) { |
| 505 /* leave the value zero: empty subsection for unused unit in a linea
r table */ |
| 506 continue; |
| 507 } |
| 508 |
| 509 /* see if there is exactly one input unit sequence of length unitIndex+1
*/ |
| 510 defaultValue=0; |
| 511 m=mappings+map[subStart]; |
| 512 if(m->bLen==unitIndex+1) { |
| 513 /* do not include this in generateToUTable() */ |
| 514 ++subStart; |
| 515 |
| 516 if(subStart<subLimit && mappings[map[subStart]].bLen==unitIndex+1) { |
| 517 /* print error for multiple same-input-sequence mappings */ |
| 518 fprintf(stderr, "error: multiple mappings from same bytes\n"); |
| 519 ucm_printMapping(table, m, stderr); |
| 520 ucm_printMapping(table, mappings+map[subStart], stderr); |
| 521 return FALSE; |
| 522 } |
| 523 |
| 524 defaultValue=getToUnicodeValue(extData, table, m); |
| 525 } |
| 526 |
| 527 if(subStart==subLimit) { |
| 528 /* write the result for the input sequence ending here */ |
| 529 section[j]|=defaultValue; |
| 530 } else { |
| 531 /* write the index to the subsection table */ |
| 532 section[j]|=(uint32_t)utm_countItems(extData->toUTable); |
| 533 |
| 534 /* recurse */ |
| 535 if(!generateToUTable(extData, table, subStart, subLimit, unitIndex+1
, defaultValue)) { |
| 536 return FALSE; |
| 537 } |
| 538 } |
| 539 } |
| 540 return TRUE; |
| 541 } |
| 542 |
| 543 /* |
| 544 * Generate the toUTable and toUUChars from the input table. |
| 545 * The input table must be sorted, and all precision flags must be 0..3. |
| 546 * This function will modify the table's reverseMap. |
| 547 */ |
| 548 static UBool |
| 549 makeToUTable(CnvExtData *extData, UCMTable *table) { |
| 550 int32_t toUCount; |
| 551 |
| 552 toUCount=reduceToUMappings(table); |
| 553 |
| 554 extData->toUTable=utm_open("cnv extension toUTable", 0x10000, UCNV_EXT_TO_U_
MIN_CODE_POINT, 4); |
| 555 extData->toUUChars=utm_open("cnv extension toUUChars", 0x10000, UCNV_EXT_TO_
U_INDEX_MASK+1, 2); |
| 556 |
| 557 return generateToUTable(extData, table, 0, toUCount, 0, 0); |
| 558 } |
| 559 |
| 560 /* from Unicode ------------------------------------------------------------- */ |
| 561 |
| 562 /* |
| 563 * preprocessing: |
| 564 * rebuild reverseMap with mapping indexes for mappings relevant for from Unicod
e |
| 565 * change each Unicode string to encode all but the first code point in 16-bit f
orm |
| 566 * |
| 567 * generation: |
| 568 * for each unique code point |
| 569 * write an entry in the 3-stage trie |
| 570 * check that there is only one single-code point sequence |
| 571 * start recursion for following 16-bit input units |
| 572 */ |
| 573 |
| 574 /* |
| 575 * Remove toUnicode fallbacks and non-<subchar1> SUB mappings |
| 576 * which are irrelevant for the fromUnicode extension table. |
| 577 * Remove MBCS_FROM_U_EXT_FLAG bits. |
| 578 * Overwrite the reverseMap with an index array to the relevant mappings. |
| 579 * Modify the code point sequences to a generator-friendly format where |
| 580 * the first code points remains unchanged but the following are recoded |
| 581 * into 16-bit Unicode string form. |
| 582 * The table must be sorted. |
| 583 * Destroys previous data in the reverseMap. |
| 584 */ |
| 585 static int32_t |
| 586 prepareFromUMappings(UCMTable *table) { |
| 587 UCMapping *mappings, *m; |
| 588 int32_t *map; |
| 589 int32_t i, j, count; |
| 590 int8_t flag; |
| 591 |
| 592 mappings=table->mappings; |
| 593 map=table->reverseMap; |
| 594 count=table->mappingsLength; |
| 595 |
| 596 /* |
| 597 * we do not go through the map on input because the mappings are |
| 598 * sorted lexically |
| 599 */ |
| 600 m=mappings; |
| 601 |
| 602 for(i=j=0; i<count; ++m, ++i) { |
| 603 flag=m->f; |
| 604 if(flag>=0) { |
| 605 flag&=MBCS_FROM_U_EXT_MASK; |
| 606 m->f=flag; |
| 607 } |
| 608 if(flag==0 || flag==1 || (flag==2 && m->bLen==1)) { |
| 609 map[j++]=i; |
| 610 |
| 611 if(m->uLen>1) { |
| 612 /* recode all but the first code point to 16-bit Unicode */ |
| 613 UChar32 *u32; |
| 614 UChar *u; |
| 615 UChar32 c; |
| 616 int32_t q, r; |
| 617 |
| 618 u32=UCM_GET_CODE_POINTS(table, m); |
| 619 u=(UChar *)u32; /* destructive in-place recoding */ |
| 620 for(r=2, q=1; q<m->uLen; ++q) { |
| 621 c=u32[q]; |
| 622 U16_APPEND_UNSAFE(u, r, c); |
| 623 } |
| 624 |
| 625 /* counts the first code point always at 2 - the first 16-bit un
it is at 16-bit index 2 */ |
| 626 m->uLen=(int8_t)r; |
| 627 } |
| 628 } |
| 629 } |
| 630 |
| 631 return j; |
| 632 } |
| 633 |
| 634 static uint32_t |
| 635 getFromUBytesValue(CnvExtData *extData, UCMTable *table, UCMapping *m) { |
| 636 uint8_t *bytes, *resultBytes; |
| 637 uint32_t value; |
| 638 int32_t u16Length, ratio; |
| 639 |
| 640 if(m->f==2) { |
| 641 /* |
| 642 * no mapping, <subchar1> preferred |
| 643 * |
| 644 * no need to count in statistics because the subchars are already |
| 645 * counted for maxOutBytes and maxBytesPerUChar in UConverterStaticData, |
| 646 * and this non-mapping does not count for maxInUChars which are always |
| 647 * trivially at least two if counting unmappable supplementary code poin
ts |
| 648 */ |
| 649 return UCNV_EXT_FROM_U_SUBCHAR1; |
| 650 } |
| 651 |
| 652 bytes=UCM_GET_BYTES(table, m); |
| 653 value=0; |
| 654 switch(m->bLen) { |
| 655 /* 1..3: store the bytes in the value word */ |
| 656 case 3: |
| 657 value=((uint32_t)*bytes++)<<16; |
| 658 case 2: |
| 659 value|=((uint32_t)*bytes++)<<8; |
| 660 case 1: |
| 661 value|=*bytes; |
| 662 break; |
| 663 default: |
| 664 /* the parser enforces m->bLen<=UCNV_EXT_MAX_BYTES */ |
| 665 /* store the bytes in fromUBytes[] and the index in the value word */ |
| 666 value=(uint32_t)utm_countItems(extData->fromUBytes); |
| 667 resultBytes=utm_allocN(extData->fromUBytes, m->bLen); |
| 668 uprv_memcpy(resultBytes, bytes, m->bLen); |
| 669 break; |
| 670 } |
| 671 value|=(uint32_t)m->bLen<<UCNV_EXT_FROM_U_LENGTH_SHIFT; |
| 672 if(m->f==0) { |
| 673 value|=UCNV_EXT_FROM_U_ROUNDTRIP_FLAG; |
| 674 } |
| 675 |
| 676 /* calculate the real UTF-16 length (see recoding in prepareFromUMappings())
*/ |
| 677 if(m->uLen==1) { |
| 678 u16Length=U16_LENGTH(m->u); |
| 679 } else { |
| 680 u16Length=U16_LENGTH(UCM_GET_CODE_POINTS(table, m)[0])+(m->uLen-2); |
| 681 } |
| 682 |
| 683 /* update statistics */ |
| 684 if(u16Length>extData->maxInUChars) { |
| 685 extData->maxInUChars=u16Length; |
| 686 } |
| 687 if(m->bLen>extData->maxOutBytes) { |
| 688 extData->maxOutBytes=m->bLen; |
| 689 } |
| 690 |
| 691 ratio=(m->bLen+(u16Length-1))/u16Length; |
| 692 if(ratio>extData->maxBytesPerUChar) { |
| 693 extData->maxBytesPerUChar=ratio; |
| 694 } |
| 695 |
| 696 return value; |
| 697 } |
| 698 |
| 699 /* |
| 700 * works like generateToUTable(), except that the |
| 701 * output section consists of two arrays, one for input UChars and one |
| 702 * for result values |
| 703 * |
| 704 * also, fromUTable sections are always stored in a compact form for |
| 705 * access via binary search |
| 706 */ |
| 707 static UBool |
| 708 generateFromUTable(CnvExtData *extData, UCMTable *table, |
| 709 int32_t start, int32_t limit, int32_t unitIndex, |
| 710 uint32_t defaultValue) { |
| 711 UCMapping *mappings, *m; |
| 712 int32_t *map; |
| 713 int32_t i, j, uniqueCount, count, subStart, subLimit; |
| 714 |
| 715 UChar *uchars; |
| 716 UChar32 low, high, prev; |
| 717 |
| 718 UChar *sectionUChars; |
| 719 uint32_t *sectionValues; |
| 720 |
| 721 mappings=table->mappings; |
| 722 map=table->reverseMap; |
| 723 |
| 724 /* step 1: examine the input units; set low, high, uniqueCount */ |
| 725 m=mappings+map[start]; |
| 726 uchars=(UChar *)UCM_GET_CODE_POINTS(table, m); |
| 727 low=uchars[unitIndex]; |
| 728 uniqueCount=1; |
| 729 |
| 730 prev=high=low; |
| 731 for(i=start+1; i<limit; ++i) { |
| 732 m=mappings+map[i]; |
| 733 uchars=(UChar *)UCM_GET_CODE_POINTS(table, m); |
| 734 high=uchars[unitIndex]; |
| 735 |
| 736 if(high!=prev) { |
| 737 prev=high; |
| 738 ++uniqueCount; |
| 739 } |
| 740 } |
| 741 |
| 742 /* step 2: allocate the section; set count, section */ |
| 743 /* the fromUTable always stores for access via binary search */ |
| 744 count=uniqueCount; |
| 745 |
| 746 /* allocate the section: 1 entry for the header + count for the items */ |
| 747 sectionUChars=(UChar *)utm_allocN(extData->fromUTableUChars, 1+count); |
| 748 sectionValues=(uint32_t *)utm_allocN(extData->fromUTableValues, 1+count); |
| 749 |
| 750 /* write the section header */ |
| 751 *sectionUChars++=(UChar)count; |
| 752 *sectionValues++=defaultValue; |
| 753 |
| 754 /* step 3: write temporary section table with subsection starts */ |
| 755 prev=low-1; /* just before low to prevent empty subsections before low */ |
| 756 j=0; /* section table index */ |
| 757 for(i=start; i<limit; ++i) { |
| 758 m=mappings+map[i]; |
| 759 uchars=(UChar *)UCM_GET_CODE_POINTS(table, m); |
| 760 high=uchars[unitIndex]; |
| 761 |
| 762 if(high!=prev) { |
| 763 /* start of a new subsection for unit high */ |
| 764 prev=high; |
| 765 |
| 766 /* write the entry with the subsection start */ |
| 767 sectionUChars[j]=(UChar)high; |
| 768 sectionValues[j]=(uint32_t)i; |
| 769 ++j; |
| 770 } |
| 771 } |
| 772 /* assert(j==count) */ |
| 773 |
| 774 /* step 4: recurse and write results */ |
| 775 subLimit=(int32_t)(sectionValues[0]); |
| 776 for(j=0; j<count; ++j) { |
| 777 subStart=subLimit; |
| 778 subLimit= (j+1)<count ? (int32_t)(sectionValues[j+1]) : limit; |
| 779 |
| 780 /* see if there is exactly one input unit sequence of length unitIndex+1
*/ |
| 781 defaultValue=0; |
| 782 m=mappings+map[subStart]; |
| 783 if(m->uLen==unitIndex+1) { |
| 784 /* do not include this in generateToUTable() */ |
| 785 ++subStart; |
| 786 |
| 787 if(subStart<subLimit && mappings[map[subStart]].uLen==unitIndex+1) { |
| 788 /* print error for multiple same-input-sequence mappings */ |
| 789 fprintf(stderr, "error: multiple mappings from same Unicode code
points\n"); |
| 790 ucm_printMapping(table, m, stderr); |
| 791 ucm_printMapping(table, mappings+map[subStart], stderr); |
| 792 return FALSE; |
| 793 } |
| 794 |
| 795 defaultValue=getFromUBytesValue(extData, table, m); |
| 796 } |
| 797 |
| 798 if(subStart==subLimit) { |
| 799 /* write the result for the input sequence ending here */ |
| 800 sectionValues[j]=defaultValue; |
| 801 } else { |
| 802 /* write the index to the subsection table */ |
| 803 sectionValues[j]=(uint32_t)utm_countItems(extData->fromUTableValues)
; |
| 804 |
| 805 /* recurse */ |
| 806 if(!generateFromUTable(extData, table, subStart, subLimit, unitIndex
+1, defaultValue)) { |
| 807 return FALSE; |
| 808 } |
| 809 } |
| 810 } |
| 811 return TRUE; |
| 812 } |
| 813 |
| 814 /* |
| 815 * add entries to the fromUnicode trie, |
| 816 * assume to be called with code points in ascending order |
| 817 * and use that to build the trie in precompacted form |
| 818 */ |
| 819 static void |
| 820 addFromUTrieEntry(CnvExtData *extData, UChar32 c, uint32_t value) { |
| 821 int32_t i1, i2, i3, i3b, nextOffset, min, newBlock; |
| 822 |
| 823 if(value==0) { |
| 824 return; |
| 825 } |
| 826 |
| 827 /* |
| 828 * compute the index for each stage, |
| 829 * allocate a stage block if necessary, |
| 830 * and write the stage value |
| 831 */ |
| 832 i1=c>>10; |
| 833 if(i1>=extData->stage1Top) { |
| 834 extData->stage1Top=i1+1; |
| 835 } |
| 836 |
| 837 nextOffset=(c>>4)&0x3f; |
| 838 |
| 839 if(extData->stage1[i1]==0) { |
| 840 /* allocate another block in stage 2; overlap with the previous block */ |
| 841 newBlock=extData->stage2Top; |
| 842 min=newBlock-nextOffset; /* minimum block start with overlap */ |
| 843 while(min<newBlock && extData->stage2[newBlock-1]==0) { |
| 844 --newBlock; |
| 845 } |
| 846 |
| 847 extData->stage1[i1]=(uint16_t)newBlock; |
| 848 extData->stage2Top=newBlock+MBCS_STAGE_2_BLOCK_SIZE; |
| 849 if(extData->stage2Top>LENGTHOF(extData->stage2)) { |
| 850 fprintf(stderr, "error: too many stage 2 entries at U+%04x\n", (int)
c); |
| 851 exit(U_MEMORY_ALLOCATION_ERROR); |
| 852 } |
| 853 } |
| 854 |
| 855 i2=extData->stage1[i1]+nextOffset; |
| 856 nextOffset=c&0xf; |
| 857 |
| 858 if(extData->stage2[i2]==0) { |
| 859 /* allocate another block in stage 3; overlap with the previous block */ |
| 860 newBlock=extData->stage3Top; |
| 861 min=newBlock-nextOffset; /* minimum block start with overlap */ |
| 862 while(min<newBlock && extData->stage3[newBlock-1]==0) { |
| 863 --newBlock; |
| 864 } |
| 865 |
| 866 /* round up to a multiple of stage 3 granularity >1 (similar to utrie.c)
*/ |
| 867 newBlock=(newBlock+(UCNV_EXT_STAGE_3_GRANULARITY-1))&~(UCNV_EXT_STAGE_3_
GRANULARITY-1); |
| 868 extData->stage2[i2]=(uint16_t)(newBlock>>UCNV_EXT_STAGE_2_LEFT_SHIFT); |
| 869 |
| 870 extData->stage3Top=newBlock+MBCS_STAGE_3_BLOCK_SIZE; |
| 871 if(extData->stage3Top>LENGTHOF(extData->stage3)) { |
| 872 fprintf(stderr, "error: too many stage 3 entries at U+%04x\n", (int)
c); |
| 873 exit(U_MEMORY_ALLOCATION_ERROR); |
| 874 } |
| 875 } |
| 876 |
| 877 i3=((int32_t)extData->stage2[i2]<<UCNV_EXT_STAGE_2_LEFT_SHIFT)+nextOffset; |
| 878 /* |
| 879 * assume extData->stage3[i3]==0 because we get |
| 880 * code points in strictly ascending order |
| 881 */ |
| 882 |
| 883 if(value==UCNV_EXT_FROM_U_SUBCHAR1) { |
| 884 /* <subchar1> SUB mapping, see getFromUBytesValue() and prepareFromUMapp
ings() */ |
| 885 extData->stage3[i3]=1; |
| 886 |
| 887 /* |
| 888 * precompaction is not optimal for <subchar1> |2 mappings because |
| 889 * stage3 values for them are all the same, unlike for other mappings |
| 890 * which all have unique values; |
| 891 * use a simple compaction of reusing a whole block filled with these |
| 892 * mappings |
| 893 */ |
| 894 |
| 895 /* is the entire block filled with <subchar1> |2 mappings? */ |
| 896 if(nextOffset==MBCS_STAGE_3_BLOCK_SIZE-1) { |
| 897 for(min=i3-nextOffset; |
| 898 min<i3 && extData->stage3[min]==1; |
| 899 ++min) {} |
| 900 |
| 901 if(min==i3) { |
| 902 /* the entire block is filled with these mappings */ |
| 903 if(extData->stage3Sub1Block!=0) { |
| 904 /* point to the previous such block and remove this block fr
om stage3 */ |
| 905 extData->stage2[i2]=extData->stage3Sub1Block; |
| 906 extData->stage3Top-=MBCS_STAGE_3_BLOCK_SIZE; |
| 907 uprv_memset(extData->stage3+extData->stage3Top, 0, MBCS_STAG
E_3_BLOCK_SIZE*2); |
| 908 } else { |
| 909 /* remember this block's stage2 entry */ |
| 910 extData->stage3Sub1Block=extData->stage2[i2]; |
| 911 } |
| 912 } |
| 913 } |
| 914 } else { |
| 915 if((i3b=extData->stage3bTop++)>=LENGTHOF(extData->stage3b)) { |
| 916 fprintf(stderr, "error: too many stage 3b entries at U+%04x\n", (int
)c); |
| 917 exit(U_MEMORY_ALLOCATION_ERROR); |
| 918 } |
| 919 |
| 920 /* roundtrip or fallback mapping */ |
| 921 extData->stage3[i3]=(uint16_t)i3b; |
| 922 extData->stage3b[i3b]=value; |
| 923 } |
| 924 } |
| 925 |
| 926 static UBool |
| 927 generateFromUTrie(CnvExtData *extData, UCMTable *table, int32_t mapLength) { |
| 928 UCMapping *mappings, *m; |
| 929 int32_t *map; |
| 930 uint32_t value; |
| 931 int32_t subStart, subLimit; |
| 932 |
| 933 UChar32 *codePoints; |
| 934 UChar32 c, next; |
| 935 |
| 936 if(mapLength==0) { |
| 937 return TRUE; |
| 938 } |
| 939 |
| 940 mappings=table->mappings; |
| 941 map=table->reverseMap; |
| 942 |
| 943 /* |
| 944 * iterate over same-initial-code point mappings, |
| 945 * enter the initial code point into the trie, |
| 946 * and start a recursion on the corresponding mappings section |
| 947 * with generateFromUTable() |
| 948 */ |
| 949 m=mappings+map[0]; |
| 950 codePoints=UCM_GET_CODE_POINTS(table, m); |
| 951 next=codePoints[0]; |
| 952 subLimit=0; |
| 953 while(subLimit<mapLength) { |
| 954 /* get a new subsection of mappings starting with the same code point */ |
| 955 subStart=subLimit; |
| 956 c=next; |
| 957 while(next==c && ++subLimit<mapLength) { |
| 958 m=mappings+map[subLimit]; |
| 959 codePoints=UCM_GET_CODE_POINTS(table, m); |
| 960 next=codePoints[0]; |
| 961 } |
| 962 |
| 963 /* |
| 964 * compute the value for this code point; |
| 965 * if there is a mapping for this code point alone, it is at subStart |
| 966 * because the table is sorted lexically |
| 967 */ |
| 968 value=0; |
| 969 m=mappings+map[subStart]; |
| 970 codePoints=UCM_GET_CODE_POINTS(table, m); |
| 971 if(m->uLen==1) { |
| 972 /* do not include this in generateFromUTable() */ |
| 973 ++subStart; |
| 974 |
| 975 if(subStart<subLimit && mappings[map[subStart]].uLen==1) { |
| 976 /* print error for multiple same-input-sequence mappings */ |
| 977 fprintf(stderr, "error: multiple mappings from same Unicode code
points\n"); |
| 978 ucm_printMapping(table, m, stderr); |
| 979 ucm_printMapping(table, mappings+map[subStart], stderr); |
| 980 return FALSE; |
| 981 } |
| 982 |
| 983 value=getFromUBytesValue(extData, table, m); |
| 984 } |
| 985 |
| 986 if(subStart==subLimit) { |
| 987 /* write the result for this one code point */ |
| 988 addFromUTrieEntry(extData, c, value); |
| 989 } else { |
| 990 /* write the index to the subsection table */ |
| 991 addFromUTrieEntry(extData, c, (uint32_t)utm_countItems(extData->from
UTableValues)); |
| 992 |
| 993 /* recurse, starting from 16-bit-unit index 2, the first 16-bit unit
after c */ |
| 994 if(!generateFromUTable(extData, table, subStart, subLimit, 2, value)
) { |
| 995 return FALSE; |
| 996 } |
| 997 } |
| 998 } |
| 999 return TRUE; |
| 1000 } |
| 1001 |
| 1002 /* |
| 1003 * Generate the fromU data structures from the input table. |
| 1004 * The input table must be sorted, and all precision flags must be 0..3. |
| 1005 * This function will modify the table's reverseMap. |
| 1006 */ |
| 1007 static UBool |
| 1008 makeFromUTable(CnvExtData *extData, UCMTable *table) { |
| 1009 uint16_t *stage1; |
| 1010 int32_t i, stage1Top, fromUCount; |
| 1011 |
| 1012 fromUCount=prepareFromUMappings(table); |
| 1013 |
| 1014 extData->fromUTableUChars=utm_open("cnv extension fromUTableUChars", 0x10000
, UCNV_EXT_FROM_U_DATA_MASK+1, 2); |
| 1015 extData->fromUTableValues=utm_open("cnv extension fromUTableValues", 0x10000
, UCNV_EXT_FROM_U_DATA_MASK+1, 4); |
| 1016 extData->fromUBytes=utm_open("cnv extension fromUBytes", 0x10000, UCNV_EXT_F
ROM_U_DATA_MASK+1, 1); |
| 1017 |
| 1018 /* allocate all-unassigned stage blocks */ |
| 1019 extData->stage2Top=MBCS_STAGE_2_FIRST_ASSIGNED; |
| 1020 extData->stage3Top=MBCS_STAGE_3_FIRST_ASSIGNED; |
| 1021 |
| 1022 /* |
| 1023 * stage 3b stores only unique values, and in |
| 1024 * index 0: 0 for "no mapping" |
| 1025 * index 1: "no mapping" with preference for <subchar1> rather than <subchar
> |
| 1026 */ |
| 1027 extData->stage3b[1]=UCNV_EXT_FROM_U_SUBCHAR1; |
| 1028 extData->stage3bTop=2; |
| 1029 |
| 1030 /* allocate the first entry in the fromUTable because index 0 means "no resu
lt" */ |
| 1031 utm_alloc(extData->fromUTableUChars); |
| 1032 utm_alloc(extData->fromUTableValues); |
| 1033 |
| 1034 if(!generateFromUTrie(extData, table, fromUCount)) { |
| 1035 return FALSE; |
| 1036 } |
| 1037 |
| 1038 /* |
| 1039 * offset the stage 1 trie entries by stage1Top because they will |
| 1040 * be stored in a single array |
| 1041 */ |
| 1042 stage1=extData->stage1; |
| 1043 stage1Top=extData->stage1Top; |
| 1044 for(i=0; i<stage1Top; ++i) { |
| 1045 stage1[i]=(uint16_t)(stage1[i]+stage1Top); |
| 1046 } |
| 1047 |
| 1048 return TRUE; |
| 1049 } |
| 1050 |
| 1051 /* -------------------------------------------------------------------------- */ |
| 1052 |
| 1053 static UBool |
| 1054 CnvExtAddTable(NewConverter *cnvData, UCMTable *table, UConverterStaticData *sta
ticData) { |
| 1055 CnvExtData *extData; |
| 1056 |
| 1057 if(table->unicodeMask&UCNV_HAS_SURROGATES) { |
| 1058 fprintf(stderr, "error: contains mappings for surrogate code points\n"); |
| 1059 return FALSE; |
| 1060 } |
| 1061 |
| 1062 staticData->conversionType=UCNV_MBCS; |
| 1063 |
| 1064 extData=(CnvExtData *)cnvData; |
| 1065 |
| 1066 /* |
| 1067 * assume that the table is sorted |
| 1068 * |
| 1069 * call the functions in this order because |
| 1070 * makeToUTable() modifies the original reverseMap, |
| 1071 * makeFromUTable() writes a whole new mapping into reverseMap |
| 1072 */ |
| 1073 return |
| 1074 makeToUTable(extData, table) && |
| 1075 makeFromUTable(extData, table); |
| 1076 } |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 5516007:
Check in the pristine copy of ICU 4.6... (Closed)
Base URL: svn://chrome-svn/chrome/trunk/deps/third_party/