Apply ICU patches for ICU tickets 6175 (ISO-2022 and ...

third_party/icu38/source/common/ucnv2022.c

 /*
 **********************************************************************
 *   Copyright (C) 2000-2007, International Business Machines
 *   Corporation and others.  All Rights Reserved.
 **********************************************************************
 *   file name:  ucnv2022.c
 *   encoding:   US-ASCII
 *   tab size:   8 (not used)
 *   indentation:4
 *
@@ skipping 183 matching lines @@
 typedef struct{
     UConverterSharedData *myConverterArray[UCNV_2022_MAX_CONVERTERS];
     UConverter *currentConverter;
     Cnv2022Type currentType;
     ISO2022State toU2022State, fromU2022State;
     uint32_t key;
     uint32_t version;
 #ifdef U_ENABLE_GENERIC_ISO_2022
     UBool isFirstBuffer;
 #endif
+    UBool isEmptySegment;
     char name[30];
     char locale[3];
 }UConverterDataISO2022;
 
 /* Protos */
 /* ISO-2022 ----------------------------------------------------------------- */
 
 /*Forward declaration */
 U_CFUNC void
 ucnv_fromUnicode_UTF8(UConverterFromUnicodeArgs * args,
@@ skipping 388 matching lines @@
         }
     }
 }
 
 static void
 _ISO2022Reset(UConverter *converter, UConverterResetChoice choice) {
     UConverterDataISO2022 *myConverterData=(UConverterDataISO2022 *) (converter->extraInfo);
     if(choice<=UCNV_RESET_TO_UNICODE) {
         uprv_memset(&myConverterData->toU2022State, 0, sizeof(ISO2022State));
         myConverterData->key = 0;
+        myConverterData->isEmptySegment = FALSE;
     }
     if(choice!=UCNV_RESET_TO_UNICODE) {
         uprv_memset(&myConverterData->fromU2022State, 0, sizeof(ISO2022State));
     }
 #ifdef U_ENABLE_GENERIC_ISO_2022
     if(myConverterData->locale[0] == 0){
         if(choice<=UCNV_RESET_TO_UNICODE) {
             myConverterData->isFirstBuffer = TRUE;
             myConverterData->key = 0;
             if (converter->mode == UCNV_SO){
@@ skipping 185 matching lines @@
         *err = U_ILLEGAL_ESCAPE_SEQUENCE;
     } else /* value == VALID_TERMINAL_2022 */ {
         switch(var){
 #ifdef U_ENABLE_GENERIC_ISO_2022
         case ISO_2022:
         {
             const char *chosenConverterName = escSeqStateTable_Result_2022[offset];
             if(chosenConverterName == NULL) {
                 /* SS2 or SS3 */
                 *err = U_UNSUPPORTED_ESCAPE_SEQUENCE;
+                _this->toUCallbackReason = UCNV_UNASSIGNED;
                 return;
             }
 
             _this->mode = UCNV_SI;
             ucnv_close(myData2022->currentConverter);
             myData2022->currentConverter = myUConverter = ucnv_open(chosenConverterName, err);
             if(U_SUCCESS(*err)) {
                 myUConverter->fromCharErrorBehaviour = UCNV_TO_U_CALLBACK_STOP;
                 _this->mode = UCNV_SO;
             }
@@ skipping 130 matching lines @@
             } else {
                 /* Back out bytes from the previous buffer: Need to replay them. */
                 _this->preToULength=(int8_t)(bytesFromThisBuffer-backOutDistance);
                 /* same as -(initialToULength-1) */
                 /* preToULength is negative! */
                 uprv_memcpy(_this->preToU, _this->toUBytes+1, -_this->preToULength);
                 *source-=bytesFromThisBuffer;
             }
             _this->toULength=1;
         }
+    } else if(*err==U_UNSUPPORTED_ESCAPE_SEQUENCE) {
+        _this->toUCallbackReason = UCNV_UNASSIGNED;
     }
 }
 
 /*Checks the characters of the buffer against valid 2022 escape sequences
 *if the match we return a pointer to the initial start of the sequence otherwise
 *we return sourceLimit
 */
 /*for 2022 looks ahead in the stream
  *to determine the longest possible convertible
  *data stream
@@ skipping 142 matching lines @@
     if(value>=0xf00) {
         return 1;  /* roundtrip */
     } else if(useFallback ? value>=0x800 : value>=0xc00) {
         return -1;  /* fallback taken */
     } else {
         return 0;  /* no mapping */
     }
 }
 
 /*
- *  * Check that the result is a 2-byte value with each byte in the range A1..FE
- *   * (strict EUC DBCS) before accepting it and subtracting 0x80 from each byte
- *    * to move it to the ISO 2022 range 21..7E.
- *     * Return 0 if out of range.
- *      */
+ * Check that the result is a 2-byte value with each byte in the range A1..FE
+ * (strict EUC DBCS) before accepting it and subtracting 0x80 from each byte
+ * to move it to the ISO 2022 range 21..7E.
+ * Return 0 if out of range.
+ */
 static U_INLINE uint32_t
 _2022FromGR94DBCS(uint32_t value) {
     if( (uint16_t)(value - 0xa1a1) <= (0xfefe - 0xa1a1) &&
         (uint8_t)(value - 0xa1) <= (0xfe - 0xa1)
     ) {
         return value - 0x8080;  /* shift down to 21..7e byte range */
     } else {
         return 0;  /* not valid for ISO 2022 */
     }
 }
 
 #if 0 /* 5691: Call sites now check for validity. They can just += 0x8080 after that. */
 /*
- * Check that the result is a 2-byte value with each byte in the range A1..FE
- * (strict EUC DBCS) before accepting it and subtracting 0x80 from each byte
- * to move it to the ISO 2022 range 21..7E.
- * Return 0 if out of range.
+ * This method does the reverse of _2022FromGR94DBCS(). Given the 2022 code point, it returns the
+ * 2 byte value that is in the range A1..FE for each byte. Otherwise it returns the 2022 code point
+ * unchanged. 
  */
 static U_INLINE uint32_t
-_2022FromGR94DBCS(uint32_t value) {
-    if( (uint16_t)(value - 0xa1a1) <= (0xfefe - 0xa1a1) &&
-        (uint8_t)(value - 0xa1) <= (0xfe - 0xa1)
-    ) {
-        return value - 0x8080;  /* shift down to 21..7e byte range */
+_2022ToGR94DBCS(uint32_t value) {
+    uint32_t returnValue = value + 0x8080;
+    if( (uint16_t)(returnValue - 0xa1a1) <= (0xfefe - 0xa1a1) &&
+        (uint8_t)(returnValue - 0xa1) <= (0xfe - 0xa1)) {
+        return returnValue;
     } else {
-        return 0;  /* not valid for ISO 2022 */
+        return value;
     }
 }
 #endif
 
 #ifdef U_ENABLE_GENERIC_ISO_2022
 
 /**********************************************************************************
 *  ISO-2022 Converter
 *
 *
@@ skipping 859 matching lines @@
 
             mySourceChar= (unsigned char) *mySource++;
 
             switch(mySourceChar) {
             case UCNV_SI:
                 if(myData->version==3) {
                     pToU2022State->g=0;
                     continue;
                 } else {
                     /* only JIS7 uses SI/SO, not ISO-2022-JP-x */
+                    myData->isEmptySegment = FALSE;     /* reset this, we have a different error */
                     break;
                 }
 
             case UCNV_SO:
                 if(myData->version==3) {
                     /* JIS7: switch to G1 half-width Katakana */
                     pToU2022State->cs[1] = (int8_t)HWKANA_7BIT;
                     pToU2022State->g=1;
                     continue;
                 } else {
                     /* only JIS7 uses SI/SO, not ISO-2022-JP-x */
+                    myData->isEmptySegment = FALSE;     /* reset this, we have a different error */
                     break;
                 }
 
             case ESC_2022:
                 mySource--;
 escape:
-                changeState_2022(args->converter,&(mySource),
-                    mySourceLimit, ISO_2022_JP,err);
+                {
+                    const char * mySourceBefore = mySource;
+                    int8_t toULengthBefore = args->converter->toULength;
+
+                    changeState_2022(args->converter,&(mySource),
+                        mySourceLimit, ISO_2022_JP,err);
+
+                    /* If in ISO-2022-JP only and we successully completed an escape sequence, but previous segment was empty, create an error */
+                    if(myData->version==0 && myData->key==0 && U_SUCCESS(*err) && myData->isEmptySegment) {
+                        *err = U_ILLEGAL_ESCAPE_SEQUENCE;
+                        args->converter->toUCallbackReason = UCNV_IRREGULAR;
+                        args->converter->toULength = toULengthBefore + (mySource - mySourceBefore);
+                    }
+                }
 
                 /* invalid or illegal escape sequence */
                 if(U_FAILURE(*err)){
                     args->target = myTarget;
                     args->source = mySource;
+                    myData->isEmptySegment = FALSE;     /* Reset to avoid future spurious errors */
                     return;
                 }
+                /* If we successfully completed an escape sequence, we begin a new segment, empty so far */
+                if(myData->key==0) {
+                    myData->isEmptySegment = TRUE;
+                }
                 continue;
 
             /* ISO-2022-JP does not use single-byte (C1) SS2 and SS3 */
 
             case CR:
                 /*falls through*/
             case LF:
                 /* automatically reset to single-byte mode */
                 if((StateEnum)pToU2022State->cs[0] != ASCII && (StateEnum)pToU2022State->cs[0] != JISX201) {
                     pToU2022State->cs[0] = (int8_t)ASCII;
                 }
                 pToU2022State->cs[2] = 0;
                 pToU2022State->g = 0;
                 /* falls through */
             default:
                 /* convert one or two bytes */
+                myData->isEmptySegment = FALSE;
                 cs = (StateEnum)pToU2022State->cs[pToU2022State->g];
                 if( (uint8_t)(mySourceChar - 0xa1) <= (0xdf - 0xa1) && myData->version==4 &&
                     !IS_JP_DBCS(cs)
                 ) {
                     /* 8-bit halfwidth katakana in any single-byte mode for JIS8 */
                     targetUniChar = mySourceChar + (HWKANA_START - 0xa1);
 
                     /* return from a single-shift state to the previous one */
                     if(pToU2022State->g >= 2) {
                         pToU2022State->g=pToU2022State->prevG;
@@ skipping 508 matching lines @@
     }
 
     while(mySource< mySourceLimit){
 
         if(myTarget < args->targetLimit){
 
             mySourceChar= (unsigned char) *mySource++;
 
             if(mySourceChar==UCNV_SI){
                 myData->toU2022State.g = 0;
+                if (myData->isEmptySegment) {
+                    myData->isEmptySegment = FALSE;     /* we are handling it, reset to avoid future spurious errors */
+                    *err = U_ILLEGAL_ESCAPE_SEQUENCE;
+                    args->converter->toUCallbackReason = UCNV_IRREGULAR;
+                    args->converter->toUBytes[0] = (uint8_t)mySourceChar;
+                    args->converter->toULength = 1;
+                    args->target = myTarget;
+                    args->source = mySource;
+                    return;
+                }
                 /*consume the source */
                 continue;
             }else if(mySourceChar==UCNV_SO){
                 myData->toU2022State.g = 1;
+                myData->isEmptySegment = TRUE;  /* Begin a new segment, empty so far */
                 /*consume the source */
                 continue;
             }else if(mySourceChar==ESC_2022){
                 mySource--;
 escape:
+                myData->isEmptySegment = FALSE; /* Any invalid ESC sequences will be detected separately, so just reset this */
                 changeState_2022(args->converter,&(mySource),
                                 mySourceLimit, ISO_2022_KR, err);
                 if(U_FAILURE(*err)){
                     args->target = myTarget;
                     args->source = mySource;
                     return;
                 }
                 continue;
             }
 
+            myData->isEmptySegment = FALSE;     /* Any invalid char errors will be detected separately, so just reset this */
             if(myData->toU2022State.g == 1) {
                 if(mySource < mySourceLimit) {
                     int leadIsOk, trailIsOk;
                     uint8_t trailByte;
 getTrailByte:
                     targetUniChar = missingCharMarker;
                     trailByte = (uint8_t)*mySource;
                     /*
                      * Ticket 5691: consistent illegal sequences:
                      * - We include at least the first byte in the illegal sequence.
@@ skipping 532 matching lines @@
 
         targetUniChar =missingCharMarker;
 
         if(myTarget < args->targetLimit){
 
             mySourceChar= (unsigned char) *mySource++;
 
             switch(mySourceChar){
             case UCNV_SI:
                 pToU2022State->g=0;
+                if (myData->isEmptySegment) {
+                    myData->isEmptySegment = FALSE;     /* we are handling it, reset to avoid future spurious errors */
+                    *err = U_ILLEGAL_ESCAPE_SEQUENCE;
+                    args->converter->toUCallbackReason = UCNV_IRREGULAR;
+                    args->converter->toUBytes[0] = mySourceChar;
+                    args->converter->toULength = 1;
+                    args->target = myTarget;
+                    args->source = mySource;
+                    return;
+                }
                 continue;
 
             case UCNV_SO:
                 if(pToU2022State->cs[1] != 0) {
                     pToU2022State->g=1;
+                    myData->isEmptySegment = TRUE;      /* Begin a new segment, empty so far */
                     continue;
                 } else {
                     /* illegal to have SO before a matching designator */
+                    myData->isEmptySegment = FALSE;     /* Handling a different error, reset this to avoid future spurious errs */
                     break;
                 }
 
             case ESC_2022:
                 mySource--;
 escape:
-                changeState_2022(args->converter,&(mySource),
-                    mySourceLimit, ISO_2022_CN,err);
+                {
+                    const char * mySourceBefore = mySource;
+                    int8_t toULengthBefore = args->converter->toULength;
+
+                    changeState_2022(args->converter,&(mySource),
+                        mySourceLimit, ISO_2022_CN,err);
+
+                    /* After SO there must be at least one character before a designator (designator error handled separately) */
+                    if(myData->key==0 && U_SUCCESS(*err) && myData->isEmptySegment) {
+                        *err = U_ILLEGAL_ESCAPE_SEQUENCE;
+                        args->converter->toUCallbackReason = UCNV_IRREGULAR;
+                        args->converter->toULength = toULengthBefore + (mySource - mySourceBefore);
+                    }
+                }
 
                 /* invalid or illegal escape sequence */
                 if(U_FAILURE(*err)){
                     args->target = myTarget;
                     args->source = mySource;
+                    myData->isEmptySegment = FALSE;     /* Reset to avoid future spurious errors */
                     return;
                 }
                 continue;
 
             /* ISO-2022-CN does not use single-byte (C1) SS2 and SS3 */
 
             case CR:
                 /*falls through*/
             case LF:
                 uprv_memset(pToU2022State, 0, sizeof(ISO2022State));
                 /* falls through */
             default:
                 /* convert one or two bytes */
+                myData->isEmptySegment = FALSE;
                 if(pToU2022State->g != 0) {
                     if(mySource < mySourceLimit) {
                         UConverterSharedData *cnv;
                         StateEnum tempState;
                         int32_t tempBufLen;
                         int leadIsOk, trailIsOk;
                         uint8_t trailByte;
 getTrailByte:
                         trailByte = (uint8_t)*mySource;
                         /*
@@ skipping 291 matching lines @@
         /* include JIS X 0201 which is hardcoded */
         sa->add(sa->set, 0xa5);
         sa->add(sa->set, 0x203e);
         if(jpCharsetMasks[cnvData->version]&CSM(ISO8859_1)) {
             /* include Latin-1 for some variants of JP */
             sa->addRange(sa->set, 0, 0xff);
         } else {
             /* include ASCII for JP */
             sa->addRange(sa->set, 0, 0x7f);
         }
-        if(jpCharsetMasks[cnvData->version]&CSM(HWKANA_7BIT)) {
+        if(cnvData->version==3 || cnvData->version==4 || which==UCNV_ROUNDTRIP_AND_FALLBACK_SET) {
             /*
-             * TODO(markus): If and when ucnv_getUnicodeSet() supports fallbacks,
-             * we need to include half-width Katakana for all JP variants because
-             * JIS X 0208 has hardcoded fallbacks for them.
+             * Do not test (jpCharsetMasks[cnvData->version]&CSM(HWKANA_7BIT))!=0
+             * because the bit is on for all JP versions although only versions 3 & 4 (JIS7 & JIS8)
+             * use half-width Katakana.
+             * This is because all ISO-2022-JP variants are lenient in that they accept (in toUnicode)
+             * half-width Katakana via the ESC ( I sequence.
+             * However, we only emit (fromUnicode) half-width Katakana according to the
+             * definition of each variant.
+             *
+             * When including fallbacks,
+             * we need to include half-width Katakana Unicode code points for all JP variants because
+             * JIS X 0208 has hardcoded fallbacks for them (which map to full-width Katakana).
              */
             /* include half-width Katakana for JP */
             sa->addRange(sa->set, HWKANA_START, HWKANA_END);
         }
         break;
     case 'c':
     case 'z':
         /* include ASCII for CN */
         sa->addRange(sa->set, 0, 0x7f);
         break;
@@ skipping 33 matching lines @@
                  * CN version 1 (-EXT) does map them all.
                  * The two versions create different Unicode sets.
                  */
                 filter=UCNV_SET_FILTER_2022_CN;
             } else if(cnvData->locale[0]=='j' && i==JISX208) {
                 /*
                  * Only add code points that map to Shift-JIS codes
                  * corresponding to JIS X 0208.
                  */
                 filter=UCNV_SET_FILTER_SJIS;
+            } else if(i==KSC5601) {
+                /*
+                 * Some of the KSC 5601 tables (convrtrs.txt has this aliases on multiple tables)
+                 * are broader than GR94.
+                 */
+                filter=UCNV_SET_FILTER_GR94DBCS;
             } else {
                 filter=UCNV_SET_FILTER_NONE;
             }
             ucnv_MBCSGetFilteredUnicodeSetForUnicode(cnvData->myConverterArray[i], sa, which, filter, pErrorCode);
         }
     }
 
     /*
      * ISO 2022 converters must not convert SO/SI/ESC despite what
      * sub-converters do by themselves.
      * Remove these characters from the set.
      */
     sa->remove(sa->set, 0x0e);
     sa->remove(sa->set, 0x0f);
     sa->remove(sa->set, 0x1b);
+
+    /* ISO 2022 converters do not convert C1 controls either */
+    sa->removeRange(sa->set, 0x80, 0x9f);
 }
 
 static const UConverterImpl _ISO2022Impl={
     UCNV_ISO_2022,
 
     NULL,
     NULL,
 
     _ISO2022Open,
     _ISO2022Close,
@@ skipping 192 matching lines @@
     NULL,
     &_ISO2022CNStaticData,
     FALSE,
     &_ISO2022CNImpl,
     0
 };
 
 
 
 #endif /* #if !UCONFIG_NO_LEGACY_CONVERSION */