ICU 56 update step 1

source/test/intltest/rbbitst.cpp

 /********************************************************************
  * COPYRIGHT:
- * Copyright (c) 1999-2014, International Business Machines Corporation and
+ * Copyright (c) 1999-2015, International Business Machines Corporation and
  * others. All Rights Reserved.
  ********************************************************************/
 /************************************************************************
 *   Date        Name        Description
 *   12/15/99    Madhu        Creation.
 *   01/12/2000  Madhu        Updated for changed API and added new tests
 ************************************************************************/
 
 #include "utypeinfo.h"  // for 'typeid' to work
 
@@ skipping 17 matching lines @@
 #include "intltest.h"
 #include "rbbitst.h"
 #include <string.h>
 #include "charstr.h"
 #include "uvector.h"
 #include "uvectr32.h"
 #include <stdio.h>
 #include <stdlib.h>
 #include "unicode/numfmt.h"
 #include "unicode/uscript.h"
+#include "cmemory.h"
+
+#if !UCONFIG_NO_FILTERED_BREAK_ITERATION
+#include "unicode/filteredbrk.h"
+#endif // !UCONFIG_NO_FILTERED_BREAK_ITERATION
 
 #define TEST_ASSERT(x) {if (!(x)) { \
     errln("Failure in file %s, line %d", __FILE__, __LINE__);}}
 
 #define TEST_ASSERT_SUCCESS(errcode) { if (U_FAILURE(errcode)) { \
     errcheckln(errcode, "Failure in file %s, line %d, status = \"%s\"", __FILE__, __LINE__, u_errorName(errcode));}}
 
 
 //---------------------------------------------
 // runIndexedTest
@@ skipping 1116 matching lines @@
 
 
 void RBBITest::TestExtended() {
 #if !UCONFIG_NO_REGULAR_EXPRESSIONS
     UErrorCode      status  = U_ZERO_ERROR;
     Locale          locale("");
 
     UnicodeString       rules;
     TestParams          tp(status);
 
-    RegexMatcher      localeMatcher(UNICODE_STRING_SIMPLE("<locale *([\\p{L}\\p{Nd}_]*) *>"), 0, status);
+    RegexMatcher      localeMatcher(UNICODE_STRING_SIMPLE("<locale *([\\p{L}\\p{Nd}_@&=-]*) *>"), 0, status);
     if (U_FAILURE(status)) {
         dataerrln("Failure in file %s, line %d, status = \"%s\"", __FILE__, __LINE__, u_errorName(status));
     }
 
 
     //
     //  Open and read the test data file.
     //
     const char *testDataDirectory = IntlTest::getSourceTestData(status);
     char testFileName[1000];
     if (testDataDirectory == NULL || strlen(testDataDirectory) >= sizeof(testFileName)) {
         errln("Can't open test data.  Path too long.");
         return;
     }
     strcpy(testFileName, testDataDirectory);
     strcat(testFileName, "rbbitst.txt");
 
     int    len;
     UChar *testFile = ReadAndConvertFile(testFileName, len, "UTF-8", status);
     if (U_FAILURE(status)) {
         return; /* something went wrong, error already output */
     }
 
 
-
+    bool skipTest = false; // Skip this test?
 
     //
     //  Put the test data into a UnicodeString
     //
     UnicodeString testString(FALSE, testFile, len);
 
     enum EParseState{
         PARSE_COMMENT,
         PARSE_TAG,
         PARSE_DATA,
@@ skipping 47 matching lines @@
                 parseState = PARSE_COMMENT;
                 savedState = PARSE_TAG;
                 break;
             }
             if (u_isUWhiteSpace(c)) {
                 break;
             }
             if (testString.compare(charIdx-1, 6, "<word>") == 0) {
                 delete tp.bi;
                 tp.bi = BreakIterator::createWordInstance(locale,  status);
+                skipTest = false;
                 charIdx += 5;
                 break;
             }
             if (testString.compare(charIdx-1, 6, "<char>") == 0) {
                 delete tp.bi;
                 tp.bi = BreakIterator::createCharacterInstance(locale,  status);
+                skipTest = false;
                 charIdx += 5;
                 break;
             }
             if (testString.compare(charIdx-1, 6, "<line>") == 0) {
                 delete tp.bi;
                 tp.bi = BreakIterator::createLineInstance(locale,  status);
+                skipTest = false;
                 charIdx += 5;
                 break;
             }
             if (testString.compare(charIdx-1, 6, "<sent>") == 0) {
                 delete tp.bi;
-                tp.bi = NULL;
                 tp.bi = BreakIterator::createSentenceInstance(locale,  status);
+                skipTest = false;
                 charIdx += 5;
                 break;
             }
             if (testString.compare(charIdx-1, 7, "<title>") == 0) {
                 delete tp.bi;
                 tp.bi = BreakIterator::createTitleInstance(locale,  status);
                 charIdx += 6;
                 break;
             }
 
@@ skipping 40 matching lines @@
             if (testString.compare(charIdx-1, 7, "</data>") == 0) {
                 // Add final entry to mappings from break location to source file position.
                 //  Need one extra because last break position returned is after the
                 //    last char in the data, not at the last char.
                 tp.srcLine->addElement(lineNum, status);
                 tp.srcCol ->addElement(column, status);
 
                 parseState = PARSE_TAG;
                 charIdx += 6;
 
-                // RUN THE TEST!
-                status = U_ZERO_ERROR;
-                tp.setUTF16(status);
-                executeTest(&tp, status);
-                TEST_ASSERT_SUCCESS(status);
+                if (!skipTest) {
+                    // RUN THE TEST!
+                    status = U_ZERO_ERROR;
+                    tp.setUTF16(status);
+                    executeTest(&tp, status);
+                    TEST_ASSERT_SUCCESS(status);
 
-                // Run again, this time with UTF-8 text wrapped in a UText.
-                status = U_ZERO_ERROR;
-                tp.setUTF8(status);
-                TEST_ASSERT_SUCCESS(status);
-                executeTest(&tp, status);
+                    // Run again, this time with UTF-8 text wrapped in a UText.
+                    status = U_ZERO_ERROR;
+                    tp.setUTF8(status);
+                    TEST_ASSERT_SUCCESS(status);
+                    executeTest(&tp, status);
+                }
                 break;
             }
 
             if (testString.compare(charIdx-1, 3, UNICODE_STRING_SIMPLE("\\N{")) == 0) {
                 // Named character, e.g. \N{COMBINING GRAVE ACCENT}
                 // Get the code point from the name and insert it into the test data.
                 //   (Damn, no API takes names in Unicode  !!!
                 //    we've got to take it back to char *)
                 int32_t nameEndIdx = testString.indexOf((UChar)0x7d/*'}'*/, charIdx);
                 int32_t nameLength = nameEndIdx - (charIdx+2);
@@ skipping 347 matching lines @@
 
     bi =  (RuleBasedBreakIterator *)BreakIterator::createLineInstance(Locale::getEnglish(), status);
     TEST_ASSERT_SUCCESS(status);
     if (U_SUCCESS(status)) {
         runUnicodeTestData("LineBreakTest.txt", bi);
     }
     delete bi;
 }
 
 
+// Check for test cases from the Unicode test data files that are known to fail
+// and should be skipped because ICU is not yet able to fully implement the spec.
+// See ticket #7270.
+
+UBool RBBITest::testCaseIsKnownIssue(const UnicodeString &testCase, const char *fileName) {
+    static const UChar badTestCases[][4] = {                     // Line Numbers from Unicode 7.0.0 file.
+        {(UChar)0x200B, (UChar)0x0020, (UChar)0x007D, (UChar)0x0000},   // Line 5198
+        {(UChar)0x200B, (UChar)0x0020, (UChar)0x0029, (UChar)0x0000},   // Line 5202
+        {(UChar)0x200B, (UChar)0x0020, (UChar)0x0021, (UChar)0x0000},   // Line 5214
+        {(UChar)0x200B, (UChar)0x0020, (UChar)0x002c, (UChar)0x0000},   // Line 5246
+        {(UChar)0x200B, (UChar)0x0020, (UChar)0x002f, (UChar)0x0000},   // Line 5298
+        {(UChar)0x200B, (UChar)0x0020, (UChar)0x2060, (UChar)0x0000}    // Line 5302
+    };
+    if (strcmp(fileName, "LineBreakTest.txt") != 0) {
+        return FALSE;
+    }
+
+    for (int i=0; i<UPRV_LENGTHOF(badTestCases); i++) {
+        if (testCase == UnicodeString(badTestCases[i])) {
+            return logKnownIssue("7270");
+        }
+    }
+    return FALSE;
+}
+
+
 //--------------------------------------------------------------------------------------------
 //
 //   Run tests from one of the boundary test data files distributed by the Unicode Consortium
 //
 //-------------------------------------------------------------------------------------------
 void RBBITest::runUnicodeTestData(const char *fileName, RuleBasedBreakIterator *bi) {
 #if !UCONFIG_NO_REGULAR_EXPRESSIONS
-    // TODO(andy): Match line break behavior to Unicode 6.0 and remove this time bomb. Ticket #7270
-    UBool isTicket7270Fixed = !logKnownIssue("7270");
-    UBool isLineBreak = 0 == strcmp(fileName, "LineBreakTest.txt");
     UErrorCode  status = U_ZERO_ERROR;
 
     //
     //  Open and read the test data file, put it into a UnicodeString.
     //
     const char *testDataDirectory = IntlTest::getSourceTestData(status);
     char testFileName[1000];
     if (testDataDirectory == NULL || strlen(testDataDirectory) >= sizeof(testFileName)) {
         dataerrln("Can't open test data.  Path too long.");
         return;
@@ skipping 71 matching lines @@
                        fileName, lineNumber);
                 }
             } else {
                 errln("Syntax Error: Hex Unicode Character value must have no more than 8 digits at \'%s\', line %d.\n",
                        fileName, lineNumber);
              }
         }
         else if (tokenMatcher.start(4, status) >= 0) {
             // Scanned to end of a line, possibly skipping over a comment in the process.
             //   If the line from the file contained test data, run the test now.
-            //
-            if (testString.length() > 0) {
-// TODO(andy): Remove this time bomb code. Note: Failing line numbers may change when updating to new Unicode data.
-//             Rule 8 
-//                ZW SP* <break>
-//             is not yet implemented.
-if (!(isLineBreak && !isTicket7270Fixed && (5198 == lineNumber || 
-                                            5202 == lineNumber ||
-                                            5214 == lineNumber ||
-                                            5246 == lineNumber ||
-                                            5298 == lineNumber ||
-                                            5302 == lineNumber ))) {
+            if (testString.length() > 0 && !testCaseIsKnownIssue(testString, fileName)) {  
                 checkUnicodeTestCase(fileName, lineNumber, testString, &breakPositions, bi);
-}
             }
 
             // Clear out this test case.
             //    The string and breakPositions vector will be refilled as the next
             //       test case is parsed.
             testString.remove();
             breakPositions.removeAllElements();
             lineNumber++;
         } else {
             // Scanner catchall.  Something unrecognized appeared on the line.
@@ skipping 884 matching lines @@
         if (p2 == prevPos) {
             // Still warming up the loop.  (won't work with zero length strings, but we don't care)
             continue;
         }
 
         // Rule (6).   ATerm x Numeric
         if (fATermSet->contains(c1) &&  fNumericSet->contains(c2))  {
             continue;
         }
 
-        // Rule (7).  Upper ATerm  x  Uppper
-        if (fUpperSet->contains(c0) && fATermSet->contains(c1) && fUpperSet->contains(c2)) {
+        // Rule (7).  (Upper | Lower) ATerm  x  Uppper
+        if ((fUpperSet->contains(c0) || fLowerSet->contains(c0)) &&
+                fATermSet->contains(c1) && fUpperSet->contains(c2)) {
             continue;
         }
 
         // Rule (8)  ATerm Close* Sp*  x  (not (OLettter | Upper | Lower | Sep | STerm | ATerm))* Lower
         //           Note:  STerm | ATerm are added to the negated part of the expression by a
         //                  note to the Unicode 5.0 documents.
         int p8 = p1;
         while (fSpSet->contains(cAt(p8))) {
             p8 = moveBack(p8);
         }
@@ skipping 594 matching lines @@
         }
 
         // LB 21b
         //   SY x HL
         if (fSY->contains(prevChar) && fHL->contains(thisChar)) {
             continue;
         }
 
         // LB 22
         if ((fAL->contains(prevChar) && fIN->contains(thisChar)) ||
+            (fEX->contains(prevChar) && fIN->contains(thisChar)) ||
             (fHL->contains(prevChar) && fIN->contains(thisChar)) ||
             (fID->contains(prevChar) && fIN->contains(thisChar)) ||
             (fIN->contains(prevChar) && fIN->contains(thisChar)) ||
             (fNU->contains(prevChar) && fIN->contains(thisChar)) )   {
             continue;
         }
 
 
         // LB 23    ID x PO
         //          AL x NU
@@ skipping 1128 matching lines @@
     UnicodeSet prependSet(UNICODE_STRING_SIMPLE("[:GCB=Prepend:]"), errorCode);
     if (!prependSet.isEmpty()) {
         errln(
             "[:GCB=Prepend:] is not empty any more. "
             "Uncomment relevant lines in source/data/brkitr/char.txt and "
             "change this test to the opposite condition.");
     }
 }
 
 #endif /* #if !UCONFIG_NO_BREAK_ITERATION */