Update ICU to 54.1 step 1

source/test/intltest/convtest.cpp

 /*
 *******************************************************************************
 *
-*   Copyright (C) 2003-2013, International Business Machines
+*   Copyright (C) 2003-2014, International Business Machines
 *   Corporation and others.  All Rights Reserved.
 *
 *******************************************************************************
 *   file name:  convtest.cpp
 *   encoding:   US-ASCII
 *   tab size:   8 (not used)
 *   indentation:4
 *
 *   created on: 2003jul15
 *   created by: Markus W. Scherer
@@ skipping 14 matching lines @@
  * not testing conversion for a custom configuration like this should be ok.
  */
 
 #include "unicode/ucnv.h"
 #include "unicode/unistr.h"
 #include "unicode/parsepos.h"
 #include "unicode/uniset.h"
 #include "unicode/ustring.h"
 #include "unicode/ures.h"
 #include "convtest.h"
+#include "cmemory.h"
 #include "unicode/tstdtmod.h"
 #include <string.h>
 #include <stdlib.h>
 
-#define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0]))
-
 enum {
     // characters used in test data for callbacks
     SUB_CB='?',
     SKIP_CB='0',
     STOP_CB='.',
     ESC_CB='&'
 };
 
 ConversionTest::ConversionTest() {
     UErrorCode errorCode=U_ZERO_ERROR;
     utf8Cnv=ucnv_open("UTF-8", &errorCode);
     ucnv_setToUCallBack(utf8Cnv, UCNV_TO_U_CALLBACK_STOP, NULL, NULL, NULL, &errorCode);
     if(U_FAILURE(errorCode)) {
         errln("unable to open UTF-8 converter");
     }
 }
 
 ConversionTest::~ConversionTest() {
     ucnv_close(utf8Cnv);
 }
 
 void
 ConversionTest::runIndexedTest(int32_t index, UBool exec, const char *&name, char * /*par*/) {
     if (exec) logln("TestSuite ConversionTest: ");
     switch (index) {
 #if !UCONFIG_NO_FILE_IO
         case 0: name="TestToUnicode"; if (exec) TestToUnicode(); break;
         case 1: name="TestFromUnicode"; if (exec) TestFromUnicode(); break;
         case 2: name="TestGetUnicodeSet"; if (exec) TestGetUnicodeSet(); break;
+        case 3: name="TestDefaultIgnorableCallback"; if (exec) TestDefaultIgnorableCallback(); break;
 #else
         case 0:
         case 1:
-        case 2: name="skip"; break;
+        case 2:
+        case 3: name="skip"; break;
 #endif
-        case 3: name="TestGetUnicodeSet2"; if (exec) TestGetUnicodeSet2(); break;
+        case 4: name="TestGetUnicodeSet2"; if (exec) TestGetUnicodeSet2(); break;
         default: name=""; break; //needed to end loop
     }
 }
 
 // test data interface ----------------------------------------------------- ***
 
 void
 ConversionTest::TestToUnicode() {
     ConversionCase cc;
     char charset[100], cbopt[4];
@@ skipping 192 matching lines @@
                         cc.subchar[j]=0;
                         cc.setSub=1;
                     }
 
                     // remove the NUL and subchar from s
                     s.truncate(index);
                 } else if((index=s.indexOf((UChar)0x3d))>0) /* '=' */ {
                     // read a substitution string, separated by an equal sign
                     p=s.getBuffer()+index+1;
                     length=s.length()-(index+1);
-                    if(length<0 || length>=LENGTHOF(cc.subString)) {
+                    if(length<0 || length>=UPRV_LENGTHOF(cc.subString)) {
                         errorCode=U_ILLEGAL_ARGUMENT_ERROR;
                     } else {
                         u_memcpy(cc.subString, p, length);
                         // NUL-terminate the subString
                         cc.subString[length]=0;
                         cc.setSub=-1;
                     }
 
                     // remove the equal sign and subString from s
                     s.truncate(index);
@@ skipping 133 matching lines @@
                             charset, i, u_errorName(errorCode));
                     errorCode=U_ZERO_ERROR;
                     continue;
                 }
 
                 // are there items that must be in cnvSet but are not?
                 (diffSet=mapSet).removeAll(cnvSet);
                 if(!diffSet.isEmpty()) {
                     diffSet.toPattern(s, TRUE);
                     if(s.length()>100) {
-                        s.replace(100, 0x7fffffff, ellipsis, LENGTHOF(ellipsis));
+                        s.replace(100, 0x7fffffff, ellipsis, UPRV_LENGTHOF(ellipsis));
                     }
                     errln("error: ucnv_getUnicodeSet(\"%s\") is missing items - conversion/getUnicodeSet test case %d",
                             charset, i);
                     errln(s);
                 }
 
                 // are there items that must not be in cnvSet but are?
                 (diffSet=mapnotSet).retainAll(cnvSet);
                 if(!diffSet.isEmpty()) {
                     diffSet.toPattern(s, TRUE);
                     if(s.length()>100) {
-                        s.replace(100, 0x7fffffff, ellipsis, LENGTHOF(ellipsis));
+                        s.replace(100, 0x7fffffff, ellipsis, UPRV_LENGTHOF(ellipsis));
                     }
                     errln("error: ucnv_getUnicodeSet(\"%s\") contains unexpected items - conversion/getUnicodeSet test case %d",
                             charset, i);
                     errln(s);
                 }
             }
             delete testData;
         }
         delete dataModule;
     }
@@ skipping 77 matching lines @@
         "HZ",
         "ISO-2022-JP",
         "JIS7",
         "ISO-2022-CN",
         "ISO-2022-CN-EXT",
         "LMBCS"
     };
     LocalUConverterPointer cnv;
     char buffer[1024];
     int32_t i;
-    for(i=0; i<LENGTHOF(cnvNames); ++i) {
+    for(i=0; i<UPRV_LENGTHOF(cnvNames); ++i) {
         UErrorCode errorCode=U_ZERO_ERROR;
         cnv.adoptInstead(cnv_open(cnvNames[i], errorCode));
         if(U_FAILURE(errorCode)) {
             errcheckln(errorCode, "failed to open converter %s - %s", cnvNames[i], u_errorName(errorCode));
             continue;
         }
         UnicodeSet expected;
         ucnv_setFromUCallBack(cnv.getAlias(), getUnicodeSetCallback, &expected, NULL, NULL, &errorCode);
         if(U_FAILURE(errorCode)) {
             errln("failed to set the callback on converter %s - %s", cnvNames[i], u_errorName(errorCode));
@@ skipping 49 matching lines @@
             }
             if(set!=expected) {
                 UnicodeSet diffSet;
                 UnicodeString out;
 
                 // are there items that must be in the set but are not?
                 (diffSet=expected).removeAll(set);
                 if(!diffSet.isEmpty()) {
                     diffSet.toPattern(out, TRUE);
                     if(out.length()>100) {
-                        out.replace(100, 0x7fffffff, ellipsis, LENGTHOF(ellipsis));
+                        out.replace(100, 0x7fffffff, ellipsis, UPRV_LENGTHOF(ellipsis));
                     }
                     errln("error: ucnv_getUnicodeSet(\"%s\") is missing items - which set: %d",
                             cnvNames[i], which);
                     errln(out);
                 }
 
                 // are there items that must not be in the set but are?
                 (diffSet=set).removeAll(expected);
                 if(!diffSet.isEmpty()) {
                     diffSet.toPattern(out, TRUE);
                     if(out.length()>100) {
-                        out.replace(100, 0x7fffffff, ellipsis, LENGTHOF(ellipsis));
+                        out.replace(100, 0x7fffffff, ellipsis, UPRV_LENGTHOF(ellipsis));
                     }
                     errln("error: ucnv_getUnicodeSet(\"%s\") contains unexpected items - which set: %d",
                             cnvNames[i], which);
                     errln(out);
                 }
             }
         }
     }
 
     delete [] s0;
 }
 
+// Test all codepoints which has the default ignorable Unicode property are ignored if they have no mapping
+// If there are any failures, the hard coded list (IS_DEFAULT_IGNORABLE_CODE_POINT) in ucnv_err.c should be updated
+void
+ConversionTest::TestDefaultIgnorableCallback() {
+    UErrorCode status = U_ZERO_ERROR;
+    const char *cnv_name = "euc-jp-2007";
+    const char *pattern_ignorable = "[:Default_Ignorable_Code_Point:]";
+    const char *pattern_not_ignorable = "[:^Default_Ignorable_Code_Point:]";
+
+    UnicodeSet *set_ignorable = new UnicodeSet(pattern_ignorable, status);
+    if (U_FAILURE(status)) {
+        dataerrln("Unable to create Unicodeset: %s - %s\n", pattern_ignorable, u_errorName(status));
+        return;
+    }
+
+    UnicodeSet *set_not_ignorable = new UnicodeSet(pattern_not_ignorable, status);
+    if (U_FAILURE(status)) {
+        dataerrln("Unable to create Unicodeset: %s - %s\n", pattern_not_ignorable, u_errorName(status));
+        return;
+    }
+
+    UConverter *cnv = cnv_open(cnv_name, status);
+    if (U_FAILURE(status)) {
+        dataerrln("Unable to open converter: %s - %s\n", cnv_name, u_errorName(status));
+        return;
+    }
+
+    // set callback for the converter 
+    ucnv_setFromUCallBack(cnv, UCNV_FROM_U_CALLBACK_SUBSTITUTE, NULL, NULL, NULL, &status);
+
+    UChar32 input[1];
+    char output[10];
+    int32_t outputLength;
+    
+    // test default ignorables are ignored
+    int size = set_ignorable->size();
+    for (int i = 0; i < size; i++) {
+        status = U_ZERO_ERROR;
+        outputLength= 0;
+
+        input[0] = set_ignorable->charAt(i);
+
+        outputLength = ucnv_fromUChars(cnv, output, 10, UnicodeString::fromUTF32(input, 1).getTerminatedBuffer(), -1, &status);
+        if (U_FAILURE(status) || outputLength != 0) {
+            errln("Ignorable code point: U+%04X not skipped as expected - %s", input[0], u_errorName(status));
+        }
+    }
+
+    // test non-ignorables are not ignored
+    size = set_not_ignorable->size();
+    for (int i = 0; i < size; i++) {
+        status = U_ZERO_ERROR;
+        outputLength= 0;
+
+        input[0] = set_not_ignorable->charAt(i);
+
+        if (input[0] == 0) {
+            continue;
+        }
+
+        outputLength = ucnv_fromUChars(cnv, output, 10, UnicodeString::fromUTF32(input, 1).getTerminatedBuffer(), -1, &status);
+        if (U_FAILURE(status) || outputLength <= 0) {
+            errln("Non-ignorable code point: U+%04X skipped unexpectedly - %s", input[0], u_errorName(status));
+        }
+    }
+    
+    ucnv_close(cnv);
+    delete set_not_ignorable;
+    delete set_ignorable;
+}
+
 // open testdata or ICU data converter ------------------------------------- ***
 
 UConverter *
 ConversionTest::cnv_open(const char *name, UErrorCode &errorCode) {
     if(name!=NULL && *name=='+') {
         // Converter names that start with '+' are ignored in ICU4J tests.
         ++name;
     }
     if(name!=NULL && *name=='*') {
         /* loadTestData(): set the data directory */
@@ skipping 322 matching lines @@
         { -2, "toU(bulk)+getNext" },
         { -3, "getNext+toU(bulk)" },
         { -4, "toU(1)+getNext" },
         { -5, "getNext+toU(1)" },
         { -12, "toU(5)+getNext" },
         { -13, "getNext+toU(5)" },
     };
     int32_t i, step;
 
     ok=TRUE;
-    for(i=0; i<LENGTHOF(steps) && ok; ++i) {
+    for(i=0; i<UPRV_LENGTHOF(steps) && ok; ++i) {
         step=steps[i].step;
         if(step<0 && !cc.finalFlush) {
             // skip ucnv_getNextUChar() if !finalFlush because
             // ucnv_getNextUChar() always implies flush
             continue;
         }
         if(step!=0) {
             // bulk test is first, then offsets are not checked any more
             cc.offsets=NULL;
         }
         else {
-            memset(resultOffsets, -1, LENGTHOF(resultOffsets));
+            memset(resultOffsets, -1, UPRV_LENGTHOF(resultOffsets));
         }
-        memset(result, -1, LENGTHOF(result));
+        memset(result, -1, UPRV_LENGTHOF(result));
         errorCode.reset();
         resultLength=stepToUnicode(cc, cnv.getAlias(),
-                                result, LENGTHOF(result),
+                                result, UPRV_LENGTHOF(result),
                                 step==0 ? resultOffsets : NULL,
                                 step, errorCode);
         ok=checkToUnicode(
                 cc, cnv.getAlias(), steps[i].name,
                 result, resultLength,
                 cc.offsets!=NULL ? resultOffsets : NULL,
                 errorCode);
         if(errorCode.isFailure() || !cc.finalFlush) {
             // reset if an error occurred or we did not flush
             // otherwise do nothing to make sure that flushing resets
             ucnv_resetToUnicode(cnv.getAlias());
         }
         if (cc.offsets != NULL && resultOffsets[resultLength] != -1) {
             errln("toUnicode[%d](%s) Conversion wrote too much to offsets at index %d",
                 cc.caseNr, cc.charset, resultLength);
         }
         if (result[resultLength] != (UChar)-1) {
             errln("toUnicode[%d](%s) Conversion wrote too much to result at index %d",
                 cc.caseNr, cc.charset, resultLength);
         }
     }
 
     // not a real loop, just a convenience for breaking out of the block
     while(ok && cc.finalFlush) {
         // test ucnv_toUChars()
         memset(result, 0, sizeof(result));
 
         errorCode.reset();
         resultLength=ucnv_toUChars(cnv.getAlias(),
-                        result, LENGTHOF(result),
+                        result, UPRV_LENGTHOF(result),
                         (const char *)cc.bytes, cc.bytesLength,
                         errorCode);
         ok=checkToUnicode(
                 cc, cnv.getAlias(), "toUChars",
                 result, resultLength,
                 NULL,
                 errorCode);
         if(!ok) {
             break;
         }
@@ skipping 126 matching lines @@
     // move only one buffer (in vs. out) at a time to be extra mean
     // step==0 performs bulk conversion
 
     // initialize the partial limits for the loop
     if(step==0) {
         // use the entire buffers
         sourceLimit=utf8Limit;
         targetLimit=resultLimit;
         flush=cc.finalFlush;
 
-        pivotLimit=pivotBuffer+LENGTHOF(pivotBuffer);
+        pivotLimit=pivotBuffer+UPRV_LENGTHOF(pivotBuffer);
     } else {
         // start with empty partial buffers
         sourceLimit=source;
         targetLimit=target;
         flush=FALSE;
 
         // empty pivot is not allowed, make it of length step
         pivotLimit=pivotBuffer+step;
     }
 
@@ skipping 198 matching lines @@
             errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d) ucnv_setSubstString() failed - %s",
                     cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, u_errorName(errorCode));
             ucnv_close(cnv);
             return FALSE;
         }
     }
 
     // convert unicode to utf8
     char utf8[256];
     cc.utf8=utf8;
-    u_strToUTF8(utf8, LENGTHOF(utf8), &cc.utf8Length,
+    u_strToUTF8(utf8, UPRV_LENGTHOF(utf8), &cc.utf8Length,
                 cc.unicode, cc.unicodeLength,
                 &errorCode);
     if(U_FAILURE(errorCode)) {
         // skip UTF-8 testing of a string with an unpaired surrogate,
         // or of one that's too long
         // toUnicode errors are tested in cintltst TestConvertExFromUTF8()
         cc.utf8Length=-1;
     }
 
     int32_t resultOffsets[256];
     char result[256];
     int32_t resultLength;
     UBool ok;
 
     static const struct {
         int32_t step;
         const char *name, *utf8Name;
     } steps[]={
         { 0, "bulk",   "utf8" }, // must be first for offsets to be checked
         { 1, "step=1", "utf8 step=1" },
         { 3, "step=3", "utf8 step=3" },
         { 7, "step=7", "utf8 step=7" }
     };
     int32_t i, step;
 
     ok=TRUE;
-    for(i=0; i<LENGTHOF(steps) && ok; ++i) {
+    for(i=0; i<UPRV_LENGTHOF(steps) && ok; ++i) {
         step=steps[i].step;
-        memset(resultOffsets, -1, LENGTHOF(resultOffsets));
-        memset(result, -1, LENGTHOF(result));
+        memset(resultOffsets, -1, UPRV_LENGTHOF(resultOffsets));
+        memset(result, -1, UPRV_LENGTHOF(result));
         errorCode=U_ZERO_ERROR;
         resultLength=stepFromUnicode(cc, cnv,
-                                result, LENGTHOF(result),
+                                result, UPRV_LENGTHOF(result),
                                 step==0 ? resultOffsets : NULL,
                                 step, &errorCode);
         ok=checkFromUnicode(
                 cc, cnv, steps[i].name,
                 (uint8_t *)result, resultLength,
                 cc.offsets!=NULL ? resultOffsets : NULL,
                 errorCode);
         if(U_FAILURE(errorCode) || !cc.finalFlush) {
             // reset if an error occurred or we did not flush
             // otherwise do nothing to make sure that flushing resets
             ucnv_resetFromUnicode(cnv);
         }
         if (resultOffsets[resultLength] != -1) {
             errln("fromUnicode[%d](%s) Conversion wrote too much to offsets at index %d",
                 cc.caseNr, cc.charset, resultLength);
         }
         if (result[resultLength] != (char)-1) {
             errln("fromUnicode[%d](%s) Conversion wrote too much to result at index %d",
                 cc.caseNr, cc.charset, resultLength);
         }
 
         // bulk test is first, then offsets are not checked any more
         cc.offsets=NULL;
 
         // test direct conversion from UTF-8
         if(cc.utf8Length>=0) {
             errorCode=U_ZERO_ERROR;
             resultLength=stepFromUTF8(cc, utf8Cnv, cnv,
-                                    result, LENGTHOF(result),
+                                    result, UPRV_LENGTHOF(result),
                                     step, &errorCode);
             ok=checkFromUnicode(
                     cc, cnv, steps[i].utf8Name,
                     (uint8_t *)result, resultLength,
                     NULL,
                     errorCode);
             if(U_FAILURE(errorCode) || !cc.finalFlush) {
                 // reset if an error occurred or we did not flush
                 // otherwise do nothing to make sure that flushing resets
                 ucnv_resetToUnicode(utf8Cnv);
                 ucnv_resetFromUnicode(cnv);
             }
         }
     }
 
     // not a real loop, just a convenience for breaking out of the block
     while(ok && cc.finalFlush) {
         // test ucnv_fromUChars()
         memset(result, 0, sizeof(result));
 
         errorCode=U_ZERO_ERROR;
         resultLength=ucnv_fromUChars(cnv,
-                        result, LENGTHOF(result),
+                        result, UPRV_LENGTHOF(result),
                         cc.unicode, cc.unicodeLength,
                         &errorCode);
         ok=checkFromUnicode(
                 cc, cnv, "fromUChars",
                 (uint8_t *)result, resultLength,
                 NULL,
                 errorCode);
         if(!ok) {
             break;
         }
@@ skipping 28 matching lines @@
     UChar resultInvalidUChars[8];
     int8_t resultInvalidLength;
     UErrorCode errorCode;
 
     const char *msg;
 
     // reset the message; NULL will mean "ok"
     msg=NULL;
 
     errorCode=U_ZERO_ERROR;
-    resultInvalidLength=LENGTHOF(resultInvalidUChars);
+    resultInvalidLength=UPRV_LENGTHOF(resultInvalidUChars);
     ucnv_getInvalidUChars(cnv, resultInvalidUChars, &resultInvalidLength, &errorCode);
     if(U_FAILURE(errorCode)) {
         errln("fromUnicode[%d](%s cb=\"%s\" fb=%d flush=%d %s) ucnv_getInvalidUChars() failed - %s",
                 cc.caseNr, cc.charset, cc.cbopt, cc.fallbacks, cc.finalFlush, name, u_errorName(errorCode));
         return FALSE;
     }
 
     // check everything that might have gone wrong
     if(cc.bytesLength!=resultLength) {
         msg="wrong result length";
@@ skipping 47 matching lines @@
               offsetsString,
               resultOffsetsString,
               u_errorName(cc.outErrorCode), u_errorName(resultErrorCode),
               invalidCharsString, resultInvalidUCharsString);
 
         return FALSE;
     }
 }
 
 #endif /* #if !UCONFIG_NO_LEGACY_CONVERSION */