Check in the pristine copy of ICU 4.6...

icu46/source/test/intltest/bidiconf.cpp

+/*
+*******************************************************************************
+*
+*   Copyright (C) 2009-2010, International Business Machines
+*   Corporation and others.  All Rights Reserved.
+*
+*******************************************************************************
+*   file name:  bidiconf.cpp
+*   encoding:   US-ASCII
+*   tab size:   8 (not used)
+*   indentation:4
+*
+*   created on: 2009oct16
+*   created by: Markus W. Scherer
+*
+*   BiDi conformance test, using the Unicode BidiTest.txt file.
+*/
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "unicode/utypes.h"
+#include "unicode/ubidi.h"
+#include "unicode/errorcode.h"
+#include "unicode/localpointer.h"
+#include "unicode/putil.h"
+#include "unicode/unistr.h"
+#include "intltest.h"
+#include "uparse.h"
+
+class BiDiConformanceTest : public IntlTest {
+public:
+    BiDiConformanceTest() :
+        directionBits(0), lineNumber(0), levelsCount(0), orderingCount(0),
+        errorCount(0) {}
+
+    void runIndexedTest(int32_t index, UBool exec, const char *&name, char *par=NULL);
+
+    void TestBidiTest();
+private:
+    char *getUnidataPath(char path[]);
+
+    UBool parseLevels(const char *start);
+    UBool parseOrdering(const char *start);
+    UBool parseInputStringFromBiDiClasses(const char *&start);
+
+    UBool checkLevels(const UBiDiLevel actualLevels[], int32_t actualCount,
+                      const char *paraLevelName);
+    UBool checkOrdering(UBiDi *ubidi, const char *paraLevelName);
+
+    void printErrorLine(const char *paraLevelName);
+
+    char line[10000];
+    UBiDiLevel levels[1000];
+    uint32_t directionBits;
+    int32_t ordering[1000];
+    int32_t lineNumber;
+    int32_t levelsCount;
+    int32_t orderingCount;
+    int32_t errorCount;
+    UnicodeString inputString;
+};
+
+extern IntlTest *createBiDiConformanceTest() {
+    return new BiDiConformanceTest();
+}
+
+void BiDiConformanceTest::runIndexedTest(int32_t index, UBool exec, const char *&name, char * /*par*/) {
+    if(exec) {
+        logln("TestSuite BiDiConformanceTest: ");
+    }
+    switch (index) {
+        TESTCASE(0, TestBidiTest);
+        default:
+            name="";
+            break; // needed to end the loop
+    }
+}
+
+// TODO: Move to a common place (IntlTest?) to avoid duplication with UnicodeTest (ucdtest.cpp).
+char *BiDiConformanceTest::getUnidataPath(char path[]) {
+    IcuTestErrorCode errorCode(*this, "getUnidataPath");
+    const int kUnicodeDataTxtLength=15;  // strlen("UnicodeData.txt")
+
+    // Look inside ICU_DATA first.
+    strcpy(path, pathToDataDirectory());
+    strcat(path, "unidata" U_FILE_SEP_STRING "UnicodeData.txt");
+    FILE *f=fopen(path, "r");
+    if(f!=NULL) {
+        fclose(f);
+        *(strchr(path, 0)-kUnicodeDataTxtLength)=0;  // Remove the basename.
+        return path;
+    }
+
+    // As a fallback, try to guess where the source data was located
+    // at the time ICU was built, and look there.
+#   ifdef U_TOPSRCDIR
+        strcpy(path, U_TOPSRCDIR  U_FILE_SEP_STRING "data");
+#   else
+        strcpy(path, loadTestData(errorCode));
+        strcat(path, U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".."
+                     U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".."
+                     U_FILE_SEP_STRING "data");
+#   endif
+    strcat(path, U_FILE_SEP_STRING);
+    strcat(path, "unidata" U_FILE_SEP_STRING "UnicodeData.txt");
+    f=fopen(path, "r");
+    if(f!=NULL) {
+        fclose(f);
+        *(strchr(path, 0)-kUnicodeDataTxtLength)=0;  // Remove the basename.
+        return path;
+    }
+    return NULL;
+}
+
+U_DEFINE_LOCAL_OPEN_POINTER(LocalStdioFilePointer, FILE, fclose);
+
+UBool BiDiConformanceTest::parseLevels(const char *start) {
+    directionBits=0;
+    levelsCount=0;
+    while(*start!=0 && *(start=u_skipWhitespace(start))!=0) {
+        if(*start=='x') {
+            levels[levelsCount++]=UBIDI_DEFAULT_LTR;
+            ++start;
+        } else {
+            char *end;
+            uint32_t value=(uint32_t)strtoul(start, &end, 10);
+            if(end<=start || (!U_IS_INV_WHITESPACE(*end) && *end!=0) || value>(UBIDI_MAX_EXPLICIT_LEVEL+1)) {
+                errln("@Levels: parse error at %s", start);
+                return FALSE;
+            }
+            levels[levelsCount++]=(UBiDiLevel)value;
+            directionBits|=(1<<(value&1));
+            start=end;
+        }
+    }
+    return TRUE;
+}
+
+UBool BiDiConformanceTest::parseOrdering(const char *start) {
+    orderingCount=0;
+    while(*start!=0 && *(start=u_skipWhitespace(start))!=0) {
+        char *end;
+        uint32_t value=(uint32_t)strtoul(start, &end, 10);
+        if(end<=start || (!U_IS_INV_WHITESPACE(*end) && *end!=0) || value>=1000) {
+            errln("@Reorder: parse error at %s", start);
+            return FALSE;
+        }
+        ordering[orderingCount++]=(int32_t)value;
+        start=end;
+    }
+    return TRUE;
+}
+
+static const UChar charFromBiDiClass[U_CHAR_DIRECTION_COUNT]={
+    0x6c,   // 'l' for L
+    0x52,   // 'R' for R
+    0x33,   // '3' for EN
+    0x2d,   // '-' for ES
+    0x25,   // '%' for ET
+    0x39,   // '9' for AN
+    0x2c,   // ',' for CS
+    0x2f,   // '/' for B
+    0x5f,   // '_' for S
+    0x20,   // ' ' for WS
+    0x3d,   // '=' for ON
+    0x65,   // 'e' for LRE
+    0x6f,   // 'o' for LRO
+    0x41,   // 'A' for AL
+    0x45,   // 'E' for RLE
+    0x4f,   // 'O' for RLO
+    0x2a,   // '*' for PDF
+    0x60,   // '`' for NSM
+    0x7c    // '|' for BN
+};
+
+U_CDECL_BEGIN
+
+static UCharDirection U_CALLCONV
+biDiConfUBiDiClassCallback(const void * /*context*/, UChar32 c) {
+    for(int i=0; i<U_CHAR_DIRECTION_COUNT; ++i) {
+        if(c==charFromBiDiClass[i]) {
+            return (UCharDirection)i;
+        }
+    }
+    // Character not in our hardcoded table.
+    // Should not occur during testing.
+    return U_BIDI_CLASS_DEFAULT;
+}
+
+U_CDECL_END
+
+static const int8_t biDiClassNameLengths[U_CHAR_DIRECTION_COUNT+1]={
+    1, 1, 2, 2, 2, 2, 2, 1, 1, 2, 2, 3, 3, 2, 3, 3, 3, 3, 2, 0
+};
+
+UBool BiDiConformanceTest::parseInputStringFromBiDiClasses(const char *&start) {
+    inputString.remove();
+    /*
+     * Lengthy but fast BiDi class parser.
+     * A simple parser could terminate or extract the name string and use
+     *   int32_t biDiClassInt=u_getPropertyValueEnum(UCHAR_BIDI_CLASS, bidiClassString);
+     * but that makes this test take significantly more time.
+     */
+    while(*start!=0 && *(start=u_skipWhitespace(start))!=0 && *start!=';') {
+        UCharDirection biDiClass=U_CHAR_DIRECTION_COUNT;
+        // Compare each character once until we have a match on
+        // a complete, short BiDi class name.
+        if(start[0]=='L') {
+            if(start[1]=='R') {
+                if(start[2]=='E') {
+                    biDiClass=U_LEFT_TO_RIGHT_EMBEDDING;
+                } else if(start[2]=='O') {
+                    biDiClass=U_LEFT_TO_RIGHT_OVERRIDE;
+                }
+            } else {
+                biDiClass=U_LEFT_TO_RIGHT;
+            }
+        } else if(start[0]=='R') {
+            if(start[1]=='L') {
+                if(start[2]=='E') {
+                    biDiClass=U_RIGHT_TO_LEFT_EMBEDDING;
+                } else if(start[2]=='O') {
+                    biDiClass=U_RIGHT_TO_LEFT_OVERRIDE;
+                }
+            } else {
+                biDiClass=U_RIGHT_TO_LEFT;
+            }
+        } else if(start[0]=='E') {
+            if(start[1]=='N') {
+                biDiClass=U_EUROPEAN_NUMBER;
+            } else if(start[1]=='S') {
+                biDiClass=U_EUROPEAN_NUMBER_SEPARATOR;
+            } else if(start[1]=='T') {
+                biDiClass=U_EUROPEAN_NUMBER_TERMINATOR;
+            }
+        } else if(start[0]=='A') {
+            if(start[1]=='L') {
+                biDiClass=U_RIGHT_TO_LEFT_ARABIC;
+            } else if(start[1]=='N') {
+                biDiClass=U_ARABIC_NUMBER;
+            }
+        } else if(start[0]=='C' && start[1]=='S') {
+            biDiClass=U_COMMON_NUMBER_SEPARATOR;
+        } else if(start[0]=='B') {
+            if(start[1]=='N') {
+                biDiClass=U_BOUNDARY_NEUTRAL;
+            } else {
+                biDiClass=U_BLOCK_SEPARATOR;
+            }
+        } else if(start[0]=='S') {
+            biDiClass=U_SEGMENT_SEPARATOR;
+        } else if(start[0]=='W' && start[1]=='S') {
+            biDiClass=U_WHITE_SPACE_NEUTRAL;
+        } else if(start[0]=='O' && start[1]=='N') {
+            biDiClass=U_OTHER_NEUTRAL;
+        } else if(start[0]=='P' && start[1]=='D' && start[2]=='F') {
+            biDiClass=U_POP_DIRECTIONAL_FORMAT;
+        } else if(start[0]=='N' && start[1]=='S' && start[2]=='M') {
+            biDiClass=U_DIR_NON_SPACING_MARK;
+        }
+        // Now we verify that the class name is terminated properly,
+        // and not just the start of a longer word.
+        int8_t biDiClassNameLength=biDiClassNameLengths[biDiClass];
+        char c=start[biDiClassNameLength];
+        if(biDiClass==U_CHAR_DIRECTION_COUNT || (!U_IS_INV_WHITESPACE(c) && c!=';' && c!=0)) {
+            errln("BiDi class string not recognized at %s", start);
+            return FALSE;
+        }
+        inputString.append(charFromBiDiClass[biDiClass]);
+        start+=biDiClassNameLength;
+    }
+    return TRUE;
+}
+
+void BiDiConformanceTest::TestBidiTest() {
+    IcuTestErrorCode errorCode(*this, "TestBidiTest");
+    const char *sourceTestDataPath=getSourceTestData(errorCode);
+    if(errorCode.logIfFailureAndReset("unable to find the source/test/testdata "
+                                      "folder (getSourceTestData())")) {
+        return;
+    }
+    char bidiTestPath[400];
+    strcpy(bidiTestPath, sourceTestDataPath);
+    strcat(bidiTestPath, "BidiTest.txt");
+    LocalStdioFilePointer bidiTestFile(fopen(bidiTestPath, "r"));
+    if(bidiTestFile.isNull()) {
+        errln("unable to open %s", bidiTestPath);
+        return;
+    }
+    LocalUBiDiPointer ubidi(ubidi_open());
+    ubidi_setClassCallback(ubidi.getAlias(), biDiConfUBiDiClassCallback, NULL,
+                           NULL, NULL, errorCode);
+    if(errorCode.logIfFailureAndReset("ubidi_setClassCallback()")) {
+        return;
+    }
+    lineNumber=0;
+    levelsCount=0;
+    orderingCount=0;
+    errorCount=0;
+    while(errorCount<10 && fgets(line, (int)sizeof(line), bidiTestFile.getAlias())!=NULL) {
+        ++lineNumber;
+        // Remove trailing comments and whitespace.
+        char *commentStart=strchr(line, '#');
+        if(commentStart!=NULL) {
+            *commentStart=0;
+        }
+        u_rtrim(line);
+        const char *start=u_skipWhitespace(line);
+        if(*start==0) {
+            continue;  // Skip empty and comment-only lines.
+        }
+        if(*start=='@') {
+            ++start;
+            if(0==strncmp(start, "Levels:", 7)) {
+                if(!parseLevels(start+7)) {
+                    return;
+                }
+            } else if(0==strncmp(start, "Reorder:", 8)) {
+                if(!parseOrdering(start+8)) {
+                    return;
+                }
+            }
+            // Skip unknown @Xyz: ...
+        } else {
+            if(!parseInputStringFromBiDiClasses(start)) {
+                return;
+            }
+            start=u_skipWhitespace(start);
+            if(*start!=';') {
+                errln("missing ; separator on input line %s", line);
+                return;
+            }
+            start=u_skipWhitespace(start+1);
+            char *end;
+            uint32_t bitset=(uint32_t)strtoul(start, &end, 16);
+            if(end<=start || (!U_IS_INV_WHITESPACE(*end) && *end!=';' && *end!=0)) {
+                errln("input bitset parse error at %s", start);
+                return;
+            }
+            // Loop over the bitset.
+            static const UBiDiLevel paraLevels[]={ UBIDI_DEFAULT_LTR, 0, 1, UBIDI_DEFAULT_RTL };
+            static const char *const paraLevelNames[]={ "auto/LTR", "LTR", "RTL", "auto/RTL" };
+            for(int i=0; i<=3; ++i) {
+                if(bitset&(1<<i)) {
+                    ubidi_setPara(ubidi.getAlias(), inputString.getBuffer(), inputString.length(),
+                                  paraLevels[i], NULL, errorCode);
+                    const UBiDiLevel *actualLevels=ubidi_getLevels(ubidi.getAlias(), errorCode);
+                    if(errorCode.logIfFailureAndReset("ubidi_setPara() or ubidi_getLevels()")) {
+                        errln("Input line %d: %s", (int)lineNumber, line);
+                        return;
+                    }
+                    if(!checkLevels(actualLevels, ubidi_getProcessedLength(ubidi.getAlias()),
+                                    paraLevelNames[i])) {
+                        // continue outerLoop;  does not exist in C++
+                        // so just break out of the inner loop.
+                        break;
+                    }
+                    if(!checkOrdering(ubidi.getAlias(), paraLevelNames[i])) {
+                        // continue outerLoop;  does not exist in C++
+                        // so just break out of the inner loop.
+                        break;
+                    }
+                }
+            }
+        }
+    }
+}
+
+static UChar printLevel(UBiDiLevel level) {
+    if(level<UBIDI_DEFAULT_LTR) {
+        return 0x30+level;
+    } else {
+        return 0x78;  // 'x'
+    }
+}
+
+static uint32_t getDirectionBits(const UBiDiLevel actualLevels[], int32_t actualCount) {
+    uint32_t actualDirectionBits=0;
+    for(int32_t i=0; i<actualCount; ++i) {
+        actualDirectionBits|=(1<<(actualLevels[i]&1));
+    }
+    return actualDirectionBits;
+}
+
+UBool BiDiConformanceTest::checkLevels(const UBiDiLevel actualLevels[], int32_t actualCount,
+                                       const char *paraLevelName) {
+    UBool isOk=TRUE;
+    if(levelsCount!=actualCount) {
+        errln("Wrong number of level values; expected %d actual %d",
+              (int)levelsCount, (int)actualCount);
+        isOk=FALSE;
+    } else {
+        for(int32_t i=0; i<actualCount; ++i) {
+            if(levels[i]!=actualLevels[i] && levels[i]<UBIDI_DEFAULT_LTR) {
+                if(directionBits!=3 && directionBits==getDirectionBits(actualLevels, actualCount)) {
+                    // ICU used a shortcut:
+                    // Since the text is unidirectional, it did not store the resolved
+                    // levels but just returns all levels as the paragraph level 0 or 1.
+                    // The reordering result is the same, so this is fine.
+                    break;
+                } else {
+                    errln("Wrong level value at index %d; expected %d actual %d",
+                          (int)i, levels[i], actualLevels[i]);
+                    isOk=FALSE;
+                    break;
+                }
+            }
+        }
+    }
+    if(!isOk) {
+        printErrorLine(paraLevelName);
+        UnicodeString els("Expected levels:   ");
+        int32_t i;
+        for(i=0; i<levelsCount; ++i) {
+            els.append((UChar)0x20).append(printLevel(levels[i]));
+        }
+        UnicodeString als("Actual   levels:   ");
+        for(i=0; i<actualCount; ++i) {
+            als.append((UChar)0x20).append(printLevel(actualLevels[i]));
+        }
+        errln(els);
+        errln(als);
+    }
+    return isOk;
+}
+
+// Note: ubidi_setReorderingOptions(ubidi, UBIDI_OPTION_REMOVE_CONTROLS);
+// does not work for custom BiDi class assignments
+// and anyway also removes LRM/RLM/ZWJ/ZWNJ which is not desirable here.
+// Therefore we just skip the indexes for BiDi controls while comparing
+// with the expected ordering that has them omitted.
+UBool BiDiConformanceTest::checkOrdering(UBiDi *ubidi, const char *paraLevelName) {
+    UBool isOk=TRUE;
+    IcuTestErrorCode errorCode(*this, "TestBidiTest/checkOrdering()");
+    int32_t resultLength=ubidi_getResultLength(ubidi);  // visual length including BiDi controls
+    int32_t i, visualIndex;
+    // Note: It should be faster to call ubidi_countRuns()/ubidi_getVisualRun()
+    // and loop over each run's indexes, but that seems unnecessary for this test code.
+    for(i=visualIndex=0; i<resultLength; ++i) {
+        int32_t logicalIndex=ubidi_getLogicalIndex(ubidi, i, errorCode);
+        if(errorCode.logIfFailureAndReset("ubidi_getLogicalIndex()")) {
+            errln("Input line %d: %s", (int)lineNumber, line);
+            return FALSE;
+        }
+        if(levels[logicalIndex]>=UBIDI_DEFAULT_LTR) {
+            continue;  // BiDi control, omitted from expected ordering.
+        }
+        if(visualIndex<orderingCount && logicalIndex!=ordering[visualIndex]) {
+            errln("Wrong ordering value at visual index %d; expected %d actual %d",
+                  (int)visualIndex, ordering[visualIndex], logicalIndex);
+            isOk=FALSE;
+            break;
+        }
+        ++visualIndex;
+    }
+    // visualIndex is now the visual length minus the BiDi controls,
+    // which should match the length of the BidiTest.txt ordering.
+    if(isOk && orderingCount!=visualIndex) {
+        errln("Wrong number of ordering values; expected %d actual %d",
+              (int)orderingCount, (int)visualIndex);
+        isOk=FALSE;
+    }
+    if(!isOk) {
+        printErrorLine(paraLevelName);
+        UnicodeString eord("Expected ordering: ");
+        for(i=0; i<orderingCount; ++i) {
+            eord.append((UChar)0x20).append((UChar)(0x30+ordering[i]));
+        }
+        UnicodeString aord("Actual   ordering: ");
+        for(i=0; i<resultLength; ++i) {
+            int32_t logicalIndex=ubidi_getLogicalIndex(ubidi, i, errorCode);
+            if(levels[logicalIndex]<UBIDI_DEFAULT_LTR) {
+                aord.append((UChar)0x20).append((UChar)(0x30+logicalIndex));
+            }
+        }
+        errln(eord);
+        errln(aord);
+    }
+    return isOk;
+}
+
+void BiDiConformanceTest::printErrorLine(const char *paraLevelName) {
+    ++errorCount;
+    errln("Input line %5d:   %s", (int)lineNumber, line);
+    errln(UnicodeString("Input string:       ")+inputString);
+    errln("Para level:         %s", paraLevelName);
+}