Apply ICU patches for ICU tickets 6175 (ISO-2022 and ...

third_party/icu38/source/common/ucnvmbcs.c

 /*
 ******************************************************************************
 *
 *   Copyright (C) 2000-2008, International Business Machines
 *   Corporation and others.  All Rights Reserved.
 *
 ******************************************************************************
 *   file name:  ucnvmbcs.c
 *   encoding:   US-ASCII
 *   tab size:   8 (not used)
@@ skipping 467 matching lines @@
     if(mbcsTable->unicodeMask&UCNV_HAS_SUPPLEMENTARY) {
         maxStage1=0x440;
     } else {
         maxStage1=0x40;
     }
 
     c=0; /* keep track of the current code point while enumerating */
 
     if(mbcsTable->outputType==MBCS_OUTPUT_1) {
         const uint16_t *stage2, *stage3, *results;
+        uint16_t minValue;
 
         results=(const uint16_t *)mbcsTable->fromUnicodeBytes;
 
+        /*
+         * Set a threshold variable for selecting which mappings to use.
+         * See ucnv_MBCSSingleFromBMPWithOffsets() and
+         * MBCS_SINGLE_RESULT_FROM_U() for details.
+         */
+        if(which==UCNV_ROUNDTRIP_SET) {
+            /* use only roundtrips */
+            minValue=0xf00;
+        } else /* UCNV_ROUNDTRIP_AND_FALLBACK_SET */ {
+            /* use all roundtrip and fallback results */
+            minValue=0x800;
+        }
+
         for(st1=0; st1<maxStage1; ++st1) {
             st2=table[st1];
             if(st2>maxStage1) {
                 stage2=table+st2;
                 for(st2=0; st2<64; ++st2) {
                     if((st3=stage2[st2])!=0) {
                         /* read the stage 3 block */
                         stage3=results+st3;
 
-                        /*
-                         * Add code points for which the roundtrip flag is set.
-                         * Once we get a set for fallback mappings, we have to use
-                         * a threshold variable with a value of 0x800.
-                         * See ucnv_MBCSSingleFromBMPWithOffsets() and
-                         * MBCS_SINGLE_RESULT_FROM_U() for details.
-                         */
                         do {
-                            if(*stage3++>=0xf00) {
+                            if(*stage3++>=minValue) {
                                 sa->add(sa->set, c);
                             }
                         } while((++c&0xf)!=0);
                     } else {
                         c+=16; /* empty stage 3 block */
                     }
                 }
             } else {
                 c+=1024; /* empty stage 2 block */
             }
         }
     } else {
         const uint32_t *stage2;
         const uint8_t *stage3, *bytes;
         uint32_t st3Multiplier;
         uint32_t value;
+        UBool useFallback;
 
         bytes=mbcsTable->fromUnicodeBytes;
 
+        useFallback=(UBool)(which==UCNV_ROUNDTRIP_AND_FALLBACK_SET);
+
         switch(mbcsTable->outputType) {
         case MBCS_OUTPUT_3:
         case MBCS_OUTPUT_4_EUC:
             st3Multiplier=3;
             break;
         case MBCS_OUTPUT_4:
             st3Multiplier=4;
             break;
         default:
             st3Multiplier=2;
             break;
         }
 
         for(st1=0; st1<maxStage1; ++st1) {
             st2=table[st1];
             if(st2>(maxStage1>>1)) {
                 stage2=(const uint32_t *)table+st2;
                 for(st2=0; st2<64; ++st2) {
                     if((st3=stage2[st2])!=0) {
                         /* read the stage 3 block */
                         stage3=bytes+st3Multiplier*16*(uint32_t)(uint16_t)st3;
 
                         /* get the roundtrip flags for the stage 3 block */
                         st3>>=16;
 
                         /*
-                         * Add code points for which the roundtrip flag is set.
-                         * Once we get a set for fallback mappings, we have to check
-                         * non-roundtrip stage 3 results for whether they are 0.
+                         * Add code points for which the roundtrip flag is set,
+                         * or which map to non-zero bytes if we use fallbacks.
                          * See ucnv_MBCSFromUnicodeWithOffsets() for details.
                          */
                         switch(filter) {
                         case UCNV_SET_FILTER_NONE:
                             do {
                                 if(st3&1) {
                                     sa->add(sa->set, c);
+                                    stage3+=st3Multiplier;
+                                } else if(useFallback) {
+                                    uint8_t b=0;
+                                    switch(st3Multiplier) {
+                                    case 4:
+                                        b|=*stage3++;
+                                    case 3:
+                                        b|=*stage3++;
+                                    case 2:
+                                        b|=stage3[0]|stage3[1];
+                                        stage3+=2;
+                                    default:
+                                        break;
+                                    }
+                                    if(b!=0) {
+                                        sa->add(sa->set, c);
+                                    }
                                 }
                                 st3>>=1;
                             } while((++c&0xf)!=0);
                             break;
                         case UCNV_SET_FILTER_DBCS_ONLY:
                              /* Ignore single-byte results (<0x100). */
                             do {
-                                if((st3&1)!=0 && *((const uint16_t *)stage3)>=0x100) {
+                                if(((st3&1)!=0 || useFallback) && *((const uint16_t *)stage3)>=0x100) {
                                     sa->add(sa->set, c);
                                 }
                                 st3>>=1;
                                 stage3+=2;  /* +=st3Multiplier */
                             } while((++c&0xf)!=0);
                             break;
                         case UCNV_SET_FILTER_2022_CN:
                              /* Only add code points that map to CNS 11643 planes 1 & 2 for non-EXT ISO-2022-CN. */
                             do {
-                                if((st3&1)!=0 && ((value=*stage3)==0x81 || value==0x82)) {
+                                if(((st3&1)!=0 || useFallback) && ((value=*stage3)==0x81 || value==0x82)) {
                                     sa->add(sa->set, c);
                                 }
                                 st3>>=1;
                                 stage3+=3;  /* +=st3Multiplier */
                             } while((++c&0xf)!=0);
                             break;
                         case UCNV_SET_FILTER_SJIS:
                              /* Only add code points that map to Shift-JIS codes corresponding to JIS X 0208. */
                             do {
-                                if((st3&1)!=0 && (value=*((const uint16_t *)stage3))>=0x8140 && value<=0xeffc) {
+                                if(((st3&1)!=0 || useFallback) && (value=*((const uint16_t *)stage3))>=0x8140 && value<=0xeffc) {
+                                    sa->add(sa->set, c);
+                                }
+                                st3>>=1;
+                                stage3+=2;  /* +=st3Multiplier */
+                            } while((++c&0xf)!=0);
+                            break;
+                        case UCNV_SET_FILTER_GR94DBCS:
+                            /* Only add code points that map to ISO 2022 GR 94 DBCS codes (each byte A1..FE). */
+                            do {
+                                if( ((st3&1)!=0 || useFallback) &&
+                                    (uint16_t)((value=*((const uint16_t *)stage3)) - 0xa1a1)<=(0xfefe - 0xa1a1) &&
+                                    (uint8_t)(value-0xa1)<=(0xfe - 0xa1)
+                                ) {
+                                    sa->add(sa->set, c);
+                                }
+                                st3>>=1;
+                                stage3+=2;  /* +=st3Multiplier */
+                            } while((++c&0xf)!=0);
+                            break;
+                        case UCNV_SET_FILTER_HZ:
+                            /* Only add code points that are suitable for HZ DBCS (lead byte A1..FD). */
+                            do {
+                                if( ((st3&1)!=0 || useFallback) &&
+                                    (uint16_t)((value=*((const uint16_t *)stage3))-0xa1a1)<=(0xfdfe - 0xa1a1) &&
+                                    (uint8_t)(value-0xa1)<=(0xfe - 0xa1)
+                                ) {
                                     sa->add(sa->set, c);
                                 }
                                 st3>>=1;
                                 stage3+=2;  /* +=st3Multiplier */
                             } while((++c&0xf)!=0);
                             break;
                         default:
                             *pErrorCode=U_INTERNAL_PROGRAM_ERROR;
                             return;
                         }
                     } else {
                         c+=16; /* empty stage 3 block */
                     }
                 }
             } else {
                 c+=1024; /* empty stage 2 block */
             }
         }
     }
 
-    ucnv_extGetUnicodeSet(sharedData, sa, which, pErrorCode);
+    ucnv_extGetUnicodeSet(sharedData, sa, which, filter, pErrorCode);
 }
 
 U_CFUNC void
 ucnv_MBCSGetUnicodeSetForUnicode(const UConverterSharedData *sharedData,
                                  const USetAdder *sa,
                                  UConverterUnicodeSet which,
                                  UErrorCode *pErrorCode) {
     ucnv_MBCSGetFilteredUnicodeSetForUnicode(
         sharedData, sa, which,
         sharedData->mbcs.outputType==MBCS_OUTPUT_DBCS_ONLY ?
@@ skipping 1064 matching lines @@
 
         if(U_FAILURE(*pErrorCode)) {
             /* callback(illegal) */
             break;
         } else /* unassigned sequences indicated with byteIndex>0 */ {
             /* try an extension mapping */
             lastSource=source;
             cnv->toUBytes[0]=*(source-1);
             cnv->toULength=_extToU(cnv, cnv->sharedData,
                                     1, &source, sourceLimit,
-                                    &target, target+targetCapacity,
+                                    &target, pArgs->targetLimit,
                                     &offsets, sourceIndex,
                                     pArgs->flush,
                                     pErrorCode);
             sourceIndex+=1+(int32_t)(source-lastSource);
 
             if(U_FAILURE(*pErrorCode)) {
                 /* not mappable or buffer overflow */
                 break;
             }
 
@@ skipping 1631 matching lines @@
                 *offsets++=sourceIndex++;
                 --count;
             }
             /* offsets and sourceIndex are now set for the current character */
         }
 
         /* try an extension mapping */
         lastSource=source;
         c=_extFromU(cnv, cnv->sharedData,
                     c, &source, sourceLimit,
-                    &target, target+targetCapacity,
+                    &target, (const uint8_t *)(pArgs->targetLimit),
                     &offsets, sourceIndex,
                     pArgs->flush,
                     pErrorCode);
         sourceIndex+=length+(int32_t)(source-lastSource);
         lastSource=source;
 
         if(U_FAILURE(*pErrorCode)) {
             /* not mappable or buffer overflow */
             break;
         } else {
@@ skipping 1720 matching lines @@
  * converters. Be sure to update it as well.
  */
 
 const UConverterSharedData _MBCSData={
     sizeof(UConverterSharedData), 1,
     NULL, NULL, NULL, FALSE, &_MBCSImpl, 
     0
 };
 
 #endif /* #if !UCONFIG_NO_LEGACY_CONVERSION */