Check in the pristine copy of ICU 4.6...

icu46/source/tools/gencnval/gencnval.c

Index: icu46/source/tools/gencnval/gencnval.c
===================================================================
--- icu46/source/tools/gencnval/gencnval.c	(revision 0)
+++ icu46/source/tools/gencnval/gencnval.c	(revision 0)
@@ -0,0 +1,1119 @@
+/*
+*******************************************************************************
+*
+*   Copyright (C) 1999-2009, International Business Machines
+*   Corporation and others.  All Rights Reserved.
+*
+*******************************************************************************
+*   file name:  gencnval.c
+*   encoding:   US-ASCII
+*   tab size:   8 (not used)
+*   indentation:4
+*
+*   created on: 1999nov05
+*   created by: Markus W. Scherer
+*
+*   This program reads convrtrs.txt and writes a memory-mappable
+*   converter name alias table to cnvalias.dat .
+*
+*   This program currently writes version 2.1 of the data format. See
+*   ucnv_io.c for more details on the format. Note that version 2.1
+*   is written in such a way that a 2.0 reader will be able to use it,
+*   and a 2.1 reader will be able to read 2.0.
+*/
+
+#include "unicode/utypes.h"
+#include "unicode/putil.h"
+#include "unicode/ucnv.h" /* ucnv_compareNames() */
+#include "ucnv_io.h"
+#include "cmemory.h"
+#include "cstring.h"
+#include "uinvchar.h"
+#include "filestrm.h"
+#include "unicode/uclean.h"
+#include "unewdata.h"
+#include "uoptions.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <ctype.h>
+
+/* TODO: Need to check alias name length is less than UCNV_MAX_CONVERTER_NAME_LENGTH */
+
+/* STRING_STORE_SIZE + TAG_STORE_SIZE <= ((2^16 - 1) * 2)
+ That is the maximum size for the string stores combined
+ because the strings are index at 16-bit boundries by a
+ 16-bit index, and there is only one section for the 
+ strings.
+ */
+#define STRING_STORE_SIZE 0x1FBFE   /* 130046 */
+#define TAG_STORE_SIZE      0x400   /* 1024 */
+
+/* The combined tag and converter count can affect the number of lists
+ created.  The size of all lists must be less than (2^17 - 1)
+ because the lists are indexed as a 16-bit array with a 16-bit index.
+ */
+#define MAX_TAG_COUNT 0x3F      /* 63 */
+#define MAX_CONV_COUNT UCNV_CONVERTER_INDEX_MASK
+#define MAX_ALIAS_COUNT 0xFFFF  /* 65535 */
+
+/* The maximum number of aliases that a standard tag/converter combination can have.
+ At this moment 6/18/2002, IANA has 12 names for ASCII. Don't go below 15 for
+ this value. I don't recommend more than 31 for this value.
+ */
+#define MAX_TC_ALIAS_COUNT 0x1F    /* 31 */
+
+#define MAX_LINE_SIZE 0x7FFF    /* 32767 */
+#define MAX_LIST_SIZE 0xFFFF    /* 65535 */
+
+#define DATA_NAME "cnvalias"
+#define DATA_TYPE "icu" /* ICU alias table */
+
+#define ALL_TAG_STR "ALL"
+#define ALL_TAG_NUM 1
+#define EMPTY_TAG_NUM 0
+
+/* UDataInfo cf. udata.h */
+static const UDataInfo dataInfo={
+    sizeof(UDataInfo),
+    0,
+
+    U_IS_BIG_ENDIAN,
+    U_CHARSET_FAMILY,
+    sizeof(UChar),
+    0,
+
+    {0x43, 0x76, 0x41, 0x6c},     /* dataFormat="CvAl" */
+    {3, 0, 1, 0},                 /* formatVersion */
+    {1, 4, 2, 0}                  /* dataVersion */
+};
+
+typedef struct {
+    char *store;
+    uint32_t top;
+    uint32_t max;
+} StringBlock;
+
+static char stringStore[STRING_STORE_SIZE];
+static StringBlock stringBlock = { stringStore, 0, STRING_STORE_SIZE };
+
+typedef struct {
+    uint16_t    aliasCount;
+    uint16_t    *aliases;     /* Index into stringStore */
+} AliasList;
+
+typedef struct {
+    uint16_t converter;     /* Index into stringStore */
+    uint16_t totalAliasCount;    /* Total aliases in this column */
+} Converter;
+
+static Converter converters[MAX_CONV_COUNT];
+static uint16_t converterCount=0;
+
+static char tagStore[TAG_STORE_SIZE];
+static StringBlock tagBlock = { tagStore, 0, TAG_STORE_SIZE };
+
+typedef struct {
+    uint16_t    tag;        /* Index into tagStore */
+    uint16_t    totalAliasCount; /* Total aliases in this row */
+    AliasList   aliasList[MAX_CONV_COUNT];
+} Tag;
+
+/* Think of this as a 3D array. It's tagCount by converterCount by aliasCount */
+static Tag tags[MAX_TAG_COUNT];
+static uint16_t tagCount = 0;
+
+/* Used for storing all aliases  */
+static uint16_t knownAliases[MAX_ALIAS_COUNT];
+static uint16_t knownAliasesCount = 0;
+/*static uint16_t duplicateKnownAliasesCount = 0;*/
+
+/* Used for storing the lists section that point to aliases */
+static uint16_t aliasLists[MAX_LIST_SIZE];
+static uint16_t aliasListsSize = 0;
+
+/* Were the standard tags declared before the aliases. */
+static UBool standardTagsUsed = FALSE;
+static UBool verbose = FALSE;
+static int lineNum = 1;
+
+static UConverterAliasOptions tableOptions = {
+    UCNV_IO_STD_NORMALIZED,
+    1 /* containsCnvOptionInfo */
+};
+
+
+/**
+ * path to convrtrs.txt
+ */
+const char *path;
+
+/* prototypes --------------------------------------------------------------- */
+
+static void
+parseLine(const char *line);
+
+static void
+parseFile(FileStream *in);
+
+static int32_t
+chomp(char *line);
+
+static void
+addOfficialTaggedStandards(char *line, int32_t lineLen);
+
+static uint16_t
+addAlias(const char *alias, uint16_t standard, uint16_t converter, UBool defaultName);
+
+static uint16_t
+addConverter(const char *converter);
+
+static char *
+allocString(StringBlock *block, const char *s, int32_t length);
+
+static uint16_t
+addToKnownAliases(const char *alias);
+
+static int
+compareAliases(const void *alias1, const void *alias2);
+
+static uint16_t
+getTagNumber(const char *tag, uint16_t tagLen);
+
+/*static void
+addTaggedAlias(uint16_t tag, const char *alias, uint16_t converter);*/
+
+static void
+writeAliasTable(UNewDataMemory *out);
+
+/* -------------------------------------------------------------------------- */
+
+/* Presumes that you used allocString() */
+#define GET_ALIAS_STR(index) (stringStore + ((size_t)(index) << 1))
+#define GET_TAG_STR(index) (tagStore + ((size_t)(index) << 1))
+
+/* Presumes that you used allocString() */
+#define GET_ALIAS_NUM(str) ((uint16_t)((str - stringStore) >> 1))
+#define GET_TAG_NUM(str) ((uint16_t)((str - tagStore) >> 1))
+
+enum
+{
+    HELP1,
+    HELP2,
+    VERBOSE,
+    COPYRIGHT,
+    DESTDIR,
+    SOURCEDIR
+};
+
+static UOption options[]={
+    UOPTION_HELP_H,
+    UOPTION_HELP_QUESTION_MARK,
+    UOPTION_VERBOSE,
+    UOPTION_COPYRIGHT,
+    UOPTION_DESTDIR,
+    UOPTION_SOURCEDIR
+};
+
+extern int
+main(int argc, char* argv[]) {
+    char pathBuf[512];
+    FileStream *in;
+    UNewDataMemory *out;
+    UErrorCode errorCode=U_ZERO_ERROR;
+
+    U_MAIN_INIT_ARGS(argc, argv);
+
+    /* preset then read command line options */
+    options[DESTDIR].value=options[SOURCEDIR].value=u_getDataDirectory();
+    argc=u_parseArgs(argc, argv, sizeof(options)/sizeof(options[0]), options);
+
+    /* error handling, printing usage message */
+    if(argc<0) {
+        fprintf(stderr,
+            "error in command line argument \"%s\"\n",
+            argv[-argc]);
+    }
+    if(argc<0 || options[HELP1].doesOccur || options[HELP2].doesOccur) {
+        fprintf(stderr,
+            "usage: %s [-options] [convrtrs.txt]\n"
+            "\tread convrtrs.txt and create " U_ICUDATA_NAME "_" DATA_NAME "." DATA_TYPE "\n"
+            "options:\n"
+            "\t-h or -? or --help  this usage text\n"
+            "\t-v or --verbose     prints out extra information about the alias table\n"
+            "\t-c or --copyright   include a copyright notice\n"
+            "\t-d or --destdir     destination directory, followed by the path\n"
+            "\t-s or --sourcedir   source directory, followed by the path\n",
+            argv[0]);
+        return argc<0 ? U_ILLEGAL_ARGUMENT_ERROR : U_ZERO_ERROR;
+    }
+
+    if(options[VERBOSE].doesOccur) {
+        verbose = TRUE;
+    }
+
+    if(argc>=2) {
+        path=argv[1];
+    } else {
+        path=options[SOURCEDIR].value;
+        if(path!=NULL && *path!=0) {
+            char *end;
+
+            uprv_strcpy(pathBuf, path);
+            end = uprv_strchr(pathBuf, 0);
+            if(*(end-1)!=U_FILE_SEP_CHAR) {
+                *(end++)=U_FILE_SEP_CHAR;
+            }
+            uprv_strcpy(end, "convrtrs.txt");
+            path=pathBuf;
+        } else {
+            path = "convrtrs.txt";
+        }
+    }
+
+    uprv_memset(stringStore, 0, sizeof(stringStore));
+    uprv_memset(tagStore, 0, sizeof(tagStore));
+    uprv_memset(converters, 0, sizeof(converters));
+    uprv_memset(tags, 0, sizeof(tags));
+    uprv_memset(aliasLists, 0, sizeof(aliasLists));
+    uprv_memset(knownAliases, 0, sizeof(aliasLists));
+
+
+    in=T_FileStream_open(path, "r");
+    if(in==NULL) {
+        fprintf(stderr, "gencnval: unable to open input file %s\n", path);
+        exit(U_FILE_ACCESS_ERROR);
+    }
+    parseFile(in);
+    T_FileStream_close(in);
+
+    /* create the output file */
+    out=udata_create(options[DESTDIR].value, DATA_TYPE, DATA_NAME, &dataInfo,
+                     options[COPYRIGHT].doesOccur ? U_COPYRIGHT_STRING : NULL, &errorCode);
+    if(U_FAILURE(errorCode)) {
+        fprintf(stderr, "gencnval: unable to open output file - error %s\n", u_errorName(errorCode));
+        exit(errorCode);
+    }
+
+    /* write the table of aliases based on a tag/converter name combination */
+    writeAliasTable(out);
+
+    /* finish */
+    udata_finish(out, &errorCode);
+    if(U_FAILURE(errorCode)) {
+        fprintf(stderr, "gencnval: error finishing output file - %s\n", u_errorName(errorCode));
+        exit(errorCode);
+    }
+
+    return 0;
+}
+
+static void
+parseFile(FileStream *in) {
+    char line[MAX_LINE_SIZE];
+    char lastLine[MAX_LINE_SIZE];
+    int32_t lineSize = 0;
+    int32_t lastLineSize = 0;
+    UBool validParse = TRUE;
+
+    lineNum = 0;
+
+    /* Add the empty tag, which is for untagged aliases */
+    getTagNumber("", 0);
+    getTagNumber(ALL_TAG_STR, 3);
+    allocString(&stringBlock, "", 0);
+
+    /* read the list of aliases */
+    while (validParse) {
+        validParse = FALSE;
+
+        /* Read non-empty lines that don't start with a space character. */
+        while (T_FileStream_readLine(in, lastLine, MAX_LINE_SIZE) != NULL) {
+            lastLineSize = chomp(lastLine);
+            if (lineSize == 0 || (lastLineSize > 0 && isspace(*lastLine))) {
+                uprv_strcpy(line + lineSize, lastLine);
+                lineSize += lastLineSize;
+            } else if (lineSize > 0) {
+                validParse = TRUE;
+                break;
+            }
+            lineNum++;
+        }
+
+        if (validParse || lineSize > 0) {
+            if (isspace(*line)) {
+                fprintf(stderr, "%s:%d: error: cannot start an alias with a space\n", path, lineNum-1);
+                exit(U_PARSE_ERROR);
+            } else if (line[0] == '{') {
+                if (!standardTagsUsed && line[lineSize - 1] != '}') {
+                    fprintf(stderr, "%s:%d: error: alias needs to start with a converter name\n", path, lineNum);
+                    exit(U_PARSE_ERROR);
+                }
+                addOfficialTaggedStandards(line, lineSize);
+                standardTagsUsed = TRUE;
+            } else {
+                if (standardTagsUsed) {
+                    parseLine(line);
+                }
+                else {
+                    fprintf(stderr, "%s:%d: error: alias table needs to start a list of standard tags\n", path, lineNum);
+                    exit(U_PARSE_ERROR);
+                }
+            }
+            /* Was the last line consumed */
+            if (lastLineSize > 0) {
+                uprv_strcpy(line, lastLine);
+                lineSize = lastLineSize;
+            }
+            else {
+                lineSize = 0;
+            }
+        }
+        lineNum++;
+    }
+}
+
+/* This works almost like the Perl chomp.
+ It removes the newlines, comments and trailing whitespace (not preceding whitespace).
+*/
+static int32_t
+chomp(char *line) {
+    char *s = line;
+    char *lastNonSpace = line;
+    while(*s!=0) {
+        /* truncate at a newline or a comment */
+        if(*s == '\r' || *s == '\n' || *s == '#') {
+            *s = 0;
+            break;
+        }
+        if (!isspace(*s)) {
+            lastNonSpace = s;
+        }
+        ++s;
+    }
+    if (lastNonSpace++ > line) {
+        *lastNonSpace = 0;
+        s = lastNonSpace;
+    }
+    return (int32_t)(s - line);
+}
+
+static void
+parseLine(const char *line) {
+    uint16_t pos=0, start, limit, length, cnv;
+    char *converter, *alias;
+
+    /* skip leading white space */
+    /* There is no whitespace at the beginning anymore */
+/*    while(line[pos]!=0 && isspace(line[pos])) {
+        ++pos;
+    }
+*/
+
+    /* is there nothing on this line? */
+    if(line[pos]==0) {
+        return;
+    }
+
+    /* get the converter name */
+    start=pos;
+    while(line[pos]!=0 && !isspace(line[pos])) {
+        ++pos;
+    }
+    limit=pos;
+
+    /* store the converter name */
+    length=(uint16_t)(limit-start);
+    converter=allocString(&stringBlock, line+start, length);
+
+    /* add the converter to the converter table */
+    cnv=addConverter(converter);
+
+    /* The name itself may be tagged, so let's added it to the aliases list properly */
+    pos = start;
+
+    /* get all the real aliases */
+    for(;;) {
+
+        /* skip white space */
+        while(line[pos]!=0 && isspace(line[pos])) {
+            ++pos;
+        }
+
+        /* is there no more alias name on this line? */
+        if(line[pos]==0) {
+            break;
+        }
+
+        /* get an alias name */
+        start=pos;
+        while(line[pos]!=0 && line[pos]!='{' && !isspace(line[pos])) {
+            ++pos;
+        }
+        limit=pos;
+
+        /* store the alias name */
+        length=(uint16_t)(limit-start);
+        if (start == 0) {
+            /* add the converter as its own alias to the alias table */
+            alias = converter;
+            addAlias(alias, ALL_TAG_NUM, cnv, TRUE);
+        }
+        else {
+            alias=allocString(&stringBlock, line+start, length);
+            addAlias(alias, ALL_TAG_NUM, cnv, FALSE);
+        }
+        addToKnownAliases(alias);
+
+        /* add the alias/converter pair to the alias table */
+        /* addAlias(alias, 0, cnv, FALSE);*/
+
+        /* skip whitespace */
+        while (line[pos] && isspace(line[pos])) {
+            ++pos;
+        }
+
+        /* handle tags if they are present */
+        if (line[pos] == '{') {
+            ++pos;
+            do {
+                start = pos;
+                while (line[pos] && line[pos] != '}' && !isspace( line[pos])) {
+                    ++pos;
+                }
+                limit = pos;
+
+                if (start != limit) {
+                    /* add the tag to the tag table */
+                    uint16_t tag = getTagNumber(line + start, (uint16_t)(limit - start));
+                    addAlias(alias, tag, cnv, (UBool)(line[limit-1] == '*'));
+                }
+
+                while (line[pos] && isspace(line[pos])) {
+                    ++pos;
+                }
+            } while (line[pos] && line[pos] != '}');
+
+            if (line[pos] == '}') {
+                ++pos;
+            } else {
+                fprintf(stderr, "%s:%d: Unterminated tag list\n", path, lineNum);
+                exit(U_UNMATCHED_BRACES);
+            }
+        } else {
+            addAlias(alias, EMPTY_TAG_NUM, cnv, (UBool)(tags[0].aliasList[cnv].aliasCount == 0));
+        }
+    }
+}
+
+static uint16_t
+getTagNumber(const char *tag, uint16_t tagLen) {
+    char *atag;
+    uint16_t t;
+    UBool preferredName = ((tagLen > 0) ? (tag[tagLen - 1] == '*') : (FALSE));
+
+    if (tagCount >= MAX_TAG_COUNT) {
+        fprintf(stderr, "%s:%d: too many tags\n", path, lineNum);
+        exit(U_BUFFER_OVERFLOW_ERROR);
+    }
+
+    if (preferredName) {
+/*        puts(tag);*/
+        tagLen--;
+    }
+
+    for (t = 0; t < tagCount; ++t) {
+        const char *currTag = GET_TAG_STR(tags[t].tag);
+        if (uprv_strlen(currTag) == tagLen && !uprv_strnicmp(currTag, tag, tagLen)) {
+            return t;
+        }
+    }
+
+    /* we need to add this tag */
+    if (tagCount >= MAX_TAG_COUNT) {
+        fprintf(stderr, "%s:%d: error: too many tags\n", path, lineNum);
+        exit(U_BUFFER_OVERFLOW_ERROR);
+    }
+
+    /* allocate a new entry in the tag table */
+    atag = allocString(&tagBlock, tag, tagLen);
+
+    if (standardTagsUsed) {
+        fprintf(stderr, "%s:%d: error: Tag \"%s\" is not declared at the beginning of the alias table.\n",
+            path, lineNum, atag);
+        exit(1);
+    }
+    else if (tagLen > 0 && strcmp(tag, ALL_TAG_STR) != 0) {
+        fprintf(stderr, "%s:%d: warning: Tag \"%s\" was added to the list of standards because it was not declared at beginning of the alias table.\n",
+            path, lineNum, atag);
+    }
+
+    /* add the tag to the tag table */
+    tags[tagCount].tag = GET_TAG_NUM(atag);
+    /* The aliasList should be set to 0's already */
+
+    return tagCount++;
+}
+
+/*static void
+addTaggedAlias(uint16_t tag, const char *alias, uint16_t converter) {
+    tags[tag].aliases[converter] = alias;
+}
+*/
+
+static void
+addOfficialTaggedStandards(char *line, int32_t lineLen) {
+    char *atag;
+    char *endTagExp;
+    char *tag;
+    static const char WHITESPACE[] = " \t";
+
+    if (tagCount > UCNV_NUM_RESERVED_TAGS) {
+        fprintf(stderr, "%s:%d: error: official tags already added\n", path, lineNum);
+        exit(U_BUFFER_OVERFLOW_ERROR);
+    }
+    tag = strchr(line, '{');
+    if (tag == NULL) {
+        /* Why were we called? */
+        fprintf(stderr, "%s:%d: error: Missing start of tag group\n", path, lineNum);
+        exit(U_PARSE_ERROR);
+    }
+    tag++;
+    endTagExp = strchr(tag, '}');
+    if (endTagExp == NULL) {
+        fprintf(stderr, "%s:%d: error: Missing end of tag group\n", path, lineNum);
+        exit(U_PARSE_ERROR);
+    }
+    endTagExp[0] = 0;
+
+    tag = strtok(tag, WHITESPACE);
+    while (tag != NULL) {
+/*        printf("Adding original tag \"%s\"\n", tag);*/
+
+        /* allocate a new entry in the tag table */
+        atag = allocString(&tagBlock, tag, -1);
+
+        /* add the tag to the tag table */
+        tags[tagCount++].tag = (uint16_t)((atag - tagStore) >> 1);
+
+        /* The aliasList should already be set to 0's */
+
+        /* Get next tag */
+        tag = strtok(NULL, WHITESPACE);
+    }
+}
+
+static uint16_t
+addToKnownAliases(const char *alias) {
+/*    uint32_t idx; */
+    /* strict matching */
+/*    for (idx = 0; idx < knownAliasesCount; idx++) {
+        uint16_t num = GET_ALIAS_NUM(alias);
+        if (knownAliases[idx] != num
+            && uprv_strcmp(alias, GET_ALIAS_STR(knownAliases[idx])) == 0)
+        {
+            fprintf(stderr, "%s:%d: warning: duplicate alias %s and %s found\n", path, 
+                lineNum, alias, GET_ALIAS_STR(knownAliases[idx]));
+            duplicateKnownAliasesCount++;
+            break;
+        }
+        else if (knownAliases[idx] != num
+            && ucnv_compareNames(alias, GET_ALIAS_STR(knownAliases[idx])) == 0)
+        {
+            if (verbose) {
+                fprintf(stderr, "%s:%d: information: duplicate alias %s and %s found\n", path, 
+                    lineNum, alias, GET_ALIAS_STR(knownAliases[idx]));
+            }
+            duplicateKnownAliasesCount++;
+            break;
+        }
+    }
+*/
+    if (knownAliasesCount >= MAX_ALIAS_COUNT) {
+        fprintf(stderr, "%s:%d: warning: Too many aliases defined for all converters\n",
+            path, lineNum);
+        exit(U_BUFFER_OVERFLOW_ERROR);
+    }
+    /* TODO: We could try to unlist exact duplicates. */
+    return knownAliases[knownAliasesCount++] = GET_ALIAS_NUM(alias);
+}
+
+/*
+@param standard When standard is 0, then it's the "empty" tag.
+*/
+static uint16_t
+addAlias(const char *alias, uint16_t standard, uint16_t converter, UBool defaultName) {
+    uint32_t idx, idx2;
+    UBool dupFound = FALSE;
+    UBool startEmptyWithoutDefault = FALSE;
+    AliasList *aliasList;
+
+    if(standard>=MAX_TAG_COUNT) {
+        fprintf(stderr, "%s:%d: error: too many standard tags\n", path, lineNum);
+        exit(U_BUFFER_OVERFLOW_ERROR);
+    }
+    if(converter>=MAX_CONV_COUNT) {
+        fprintf(stderr, "%s:%d: error: too many converter names\n", path, lineNum);
+        exit(U_BUFFER_OVERFLOW_ERROR);
+    }
+    aliasList = &tags[standard].aliasList[converter];
+
+    if (strchr(alias, '}')) {
+        fprintf(stderr, "%s:%d: error: unmatched } found\n", path, 
+            lineNum);
+    }
+
+    if(aliasList->aliasCount + 1 >= MAX_TC_ALIAS_COUNT) {
+        fprintf(stderr, "%s:%d: error: too many aliases for alias %s and converter %s\n", path, 
+            lineNum, alias, GET_ALIAS_STR(converters[converter].converter));
+        exit(U_BUFFER_OVERFLOW_ERROR);
+    }
+
+    /* Show this warning only once. All aliases are added to the "ALL" tag. */
+    if (standard == ALL_TAG_NUM && GET_ALIAS_STR(converters[converter].converter) != alias) {
+        /* Normally these option values are parsed at runtime, and they can
+           be discarded when the alias is a default converter. Options should
+           only be on a converter and not an alias. */
+        if (uprv_strchr(alias, UCNV_OPTION_SEP_CHAR) != 0)
+        {
+            fprintf(stderr, "warning(line %d): alias %s contains a \""UCNV_OPTION_SEP_STRING"\". Options are parsed at run-time and do not need to be in the alias table.\n",
+                lineNum, alias);
+        }
+        if (uprv_strchr(alias, UCNV_VALUE_SEP_CHAR) != 0)
+        {
+            fprintf(stderr, "warning(line %d): alias %s contains an \""UCNV_VALUE_SEP_STRING"\". Options are parsed at run-time and do not need to be in the alias table.\n",
+                lineNum, alias);
+        }
+    }
+
+    if (standard != ALL_TAG_NUM) {
+        /* Check for duplicate aliases for this tag on all converters */
+        for (idx = 0; idx < converterCount; idx++) {
+            for (idx2 = 0; idx2 < tags[standard].aliasList[idx].aliasCount; idx2++) {
+                uint16_t aliasNum = tags[standard].aliasList[idx].aliases[idx2];
+                if (aliasNum
+                    && ucnv_compareNames(alias, GET_ALIAS_STR(aliasNum)) == 0)
+                {
+                    if (idx == converter) {
+                        /*
+                         * (alias, standard) duplicates are harmless if they map to the same converter.
+                         * Only print a warning in verbose mode, or if the alias is a precise duplicate,
+                         * not just a lenient-match duplicate.
+                         */
+                        if (verbose || 0 == uprv_strcmp(alias, GET_ALIAS_STR(aliasNum))) {
+                            fprintf(stderr, "%s:%d: warning: duplicate aliases %s and %s found for standard %s and converter %s\n", path, 
+                                lineNum, alias, GET_ALIAS_STR(aliasNum),
+                                GET_TAG_STR(tags[standard].tag),
+                                GET_ALIAS_STR(converters[converter].converter));
+                        }
+                    } else {
+                        fprintf(stderr, "%s:%d: warning: duplicate aliases %s and %s found for standard tag %s between converter %s and converter %s\n", path, 
+                            lineNum, alias, GET_ALIAS_STR(aliasNum),
+                            GET_TAG_STR(tags[standard].tag),
+                            GET_ALIAS_STR(converters[converter].converter),
+                            GET_ALIAS_STR(converters[idx].converter));
+                    }
+                    dupFound = TRUE;
+                    break;
+                }
+            }
+        }
+
+        /* Check for duplicate default aliases for this converter on all tags */
+        /* It's okay to have multiple standards prefer the same name */
+/*        if (verbose && !dupFound) {
+            for (idx = 0; idx < tagCount; idx++) {
+                if (tags[idx].aliasList[converter].aliases) {
+                    uint16_t aliasNum = tags[idx].aliasList[converter].aliases[0];
+                    if (aliasNum
+                        && ucnv_compareNames(alias, GET_ALIAS_STR(aliasNum)) == 0)
+                    {
+                        fprintf(stderr, "%s:%d: warning: duplicate alias %s found for converter %s and standard tag %s\n", path, 
+                            lineNum, alias, GET_ALIAS_STR(converters[converter].converter), GET_TAG_STR(tags[standard].tag));
+                        break;
+                    }
+                }
+            }
+        }*/
+    }
+
+    if (aliasList->aliasCount <= 0) {
+        aliasList->aliasCount++;
+        startEmptyWithoutDefault = TRUE;
+    }
+    aliasList->aliases = (uint16_t *)uprv_realloc(aliasList->aliases, (aliasList->aliasCount + 1) * sizeof(aliasList->aliases[0]));
+    if (startEmptyWithoutDefault) {
+        aliasList->aliases[0] = 0;
+    }
+    if (defaultName) {
+        if (aliasList->aliases[0] != 0) {
+            fprintf(stderr, "%s:%d: error: Alias %s and %s cannot both be the default alias for standard tag %s and converter %s\n", path, 
+                lineNum,
+                alias,
+                GET_ALIAS_STR(aliasList->aliases[0]),
+                GET_TAG_STR(tags[standard].tag),
+                GET_ALIAS_STR(converters[converter].converter));
+            exit(U_PARSE_ERROR);
+        }
+        aliasList->aliases[0] = GET_ALIAS_NUM(alias);
+    } else {
+        aliasList->aliases[aliasList->aliasCount++] = GET_ALIAS_NUM(alias);
+    }
+/*    aliasList->converter = converter;*/
+
+    converters[converter].totalAliasCount++; /* One more to the column */
+    tags[standard].totalAliasCount++; /* One more to the row */
+
+    return aliasList->aliasCount;
+}
+
+static uint16_t
+addConverter(const char *converter) {
+    uint32_t idx;
+    if(converterCount>=MAX_CONV_COUNT) {
+        fprintf(stderr, "%s:%d: error: too many converters\n", path, lineNum);
+        exit(U_BUFFER_OVERFLOW_ERROR);
+    }
+
+    for (idx = 0; idx < converterCount; idx++) {
+        if (ucnv_compareNames(converter, GET_ALIAS_STR(converters[idx].converter)) == 0) {
+            fprintf(stderr, "%s:%d: error: duplicate converter %s found!\n", path, lineNum, converter);
+            exit(U_PARSE_ERROR);
+            break;
+        }
+    }
+
+    converters[converterCount].converter = GET_ALIAS_NUM(converter);
+    converters[converterCount].totalAliasCount = 0;
+
+    return converterCount++;
+}
+
+/* resolve this alias based on the prioritization of the standard tags. */
+static void
+resolveAliasToConverter(uint16_t alias, uint16_t *tagNum, uint16_t *converterNum) {
+    uint16_t idx, idx2, idx3;
+
+    for (idx = UCNV_NUM_RESERVED_TAGS; idx < tagCount; idx++) {
+        for (idx2 = 0; idx2 < converterCount; idx2++) {
+            for (idx3 = 0; idx3 < tags[idx].aliasList[idx2].aliasCount; idx3++) {
+                uint16_t aliasNum = tags[idx].aliasList[idx2].aliases[idx3];
+                if (aliasNum == alias) {
+                    *tagNum = idx;
+                    *converterNum = idx2;
+                    return;
+                }
+            }
+        }
+    }
+    /* Do the leftovers last, just in case */
+    /* There is no need to do the ALL tag */
+    idx = 0;
+    for (idx2 = 0; idx2 < converterCount; idx2++) {
+        for (idx3 = 0; idx3 < tags[idx].aliasList[idx2].aliasCount; idx3++) {
+            uint16_t aliasNum = tags[idx].aliasList[idx2].aliases[idx3];
+            if (aliasNum == alias) {
+                *tagNum = idx;
+                *converterNum = idx2;
+                return;
+            }
+        }
+    }
+    *tagNum = UINT16_MAX;
+    *converterNum = UINT16_MAX;
+    fprintf(stderr, "%s: warning: alias %s not found\n",
+        path,
+        GET_ALIAS_STR(alias));
+    return;
+}
+
+/* The knownAliases should be sorted before calling this function */
+static uint32_t
+resolveAliases(uint16_t *uniqueAliasArr, uint16_t *uniqueAliasToConverterArr, uint16_t aliasOffset) {
+    uint32_t uniqueAliasIdx = 0;
+    uint32_t idx;
+    uint16_t currTagNum, oldTagNum;
+    uint16_t currConvNum, oldConvNum;
+    const char *lastName;
+
+    resolveAliasToConverter(knownAliases[0], &oldTagNum, &currConvNum);
+    uniqueAliasToConverterArr[uniqueAliasIdx] = currConvNum;
+    oldConvNum = currConvNum;
+    uniqueAliasArr[uniqueAliasIdx] = knownAliases[0] + aliasOffset;
+    uniqueAliasIdx++;
+    lastName = GET_ALIAS_STR(knownAliases[0]);
+
+    for (idx = 1; idx < knownAliasesCount; idx++) {
+        resolveAliasToConverter(knownAliases[idx], &currTagNum, &currConvNum);
+        if (ucnv_compareNames(lastName, GET_ALIAS_STR(knownAliases[idx])) == 0) {
+            /* duplicate found */
+            if ((currTagNum < oldTagNum && currTagNum >= UCNV_NUM_RESERVED_TAGS)
+                || oldTagNum == 0) {
+                oldTagNum = currTagNum;
+                uniqueAliasToConverterArr[uniqueAliasIdx - 1] = currConvNum;
+                uniqueAliasArr[uniqueAliasIdx - 1] = knownAliases[idx] + aliasOffset;
+                if (verbose) {
+                    printf("using %s instead of %s -> %s", 
+                        GET_ALIAS_STR(knownAliases[idx]),
+                        lastName,
+                        GET_ALIAS_STR(converters[currConvNum].converter));
+                    if (oldConvNum != currConvNum) {
+                        printf(" (alias conflict)");
+                    }
+                    puts("");
+                }
+            }
+            else {
+                /* else ignore it */
+                if (verbose) {
+                    printf("folding %s into %s -> %s",
+                        GET_ALIAS_STR(knownAliases[idx]),
+                        lastName,
+                        GET_ALIAS_STR(converters[oldConvNum].converter));
+                    if (oldConvNum != currConvNum) {
+                        printf(" (alias conflict)");
+                    }
+                    puts("");
+                }
+            }
+            if (oldConvNum != currConvNum) {
+                uniqueAliasToConverterArr[uniqueAliasIdx - 1] |= UCNV_AMBIGUOUS_ALIAS_MAP_BIT;
+            }
+        }
+        else {
+            uniqueAliasToConverterArr[uniqueAliasIdx] = currConvNum;
+            oldConvNum = currConvNum;
+            uniqueAliasArr[uniqueAliasIdx] = knownAliases[idx] + aliasOffset;
+            uniqueAliasIdx++;
+            lastName = GET_ALIAS_STR(knownAliases[idx]);
+            oldTagNum = currTagNum;
+            /*printf("%s -> %s\n", GET_ALIAS_STR(knownAliases[idx]), GET_ALIAS_STR(converters[currConvNum].converter));*/
+        }
+        if (uprv_strchr(GET_ALIAS_STR(converters[currConvNum].converter), UCNV_OPTION_SEP_CHAR) != NULL) {
+            uniqueAliasToConverterArr[uniqueAliasIdx-1] |= UCNV_CONTAINS_OPTION_BIT;
+        }
+    }
+    return uniqueAliasIdx;
+}
+
+static void
+createOneAliasList(uint16_t *aliasArrLists, uint32_t tag, uint32_t converter, uint16_t offset) {
+    uint32_t aliasNum;
+    AliasList *aliasList = &tags[tag].aliasList[converter];
+
+    if (aliasList->aliasCount == 0) {
+        aliasArrLists[tag*converterCount + converter] = 0;
+    }
+    else {
+        aliasLists[aliasListsSize++] = aliasList->aliasCount;
+
+        /* write into the array area a 1's based index. */
+        aliasArrLists[tag*converterCount + converter] = aliasListsSize;
+
+/*        printf("tag %s converter %s\n",
+            GET_TAG_STR(tags[tag].tag),
+            GET_ALIAS_STR(converters[converter].converter));*/
+        for (aliasNum = 0; aliasNum < aliasList->aliasCount; aliasNum++) {
+            uint16_t value;
+/*            printf("   %s\n",
+                GET_ALIAS_STR(aliasList->aliases[aliasNum]));*/
+            if (aliasList->aliases[aliasNum]) {
+                value = aliasList->aliases[aliasNum] + offset;
+            } else {
+                value = 0;
+                if (tag != 0) { /* Only show the warning when it's not the leftover tag. */
+                    fprintf(stderr, "%s: warning: tag %s does not have a default alias for %s\n",
+                            path,
+                            GET_TAG_STR(tags[tag].tag),
+                            GET_ALIAS_STR(converters[converter].converter));
+                }
+            }
+            aliasLists[aliasListsSize++] = value;
+            if (aliasListsSize >= MAX_LIST_SIZE) {
+                fprintf(stderr, "%s: error: Too many alias lists\n", path);
+                exit(U_BUFFER_OVERFLOW_ERROR);
+            }
+
+        }
+    }
+}
+
+static void
+createNormalizedAliasStrings(char *normalizedStrings, const char *origStringBlock, int32_t stringBlockLength) {
+    int32_t currStrLen;
+    uprv_memcpy(normalizedStrings, origStringBlock, stringBlockLength);
+    while ((currStrLen = (int32_t)uprv_strlen(origStringBlock)) < stringBlockLength) {
+        int32_t currStrSize = currStrLen + 1;
+        if (currStrLen > 0) {
+            int32_t normStrLen;
+            ucnv_io_stripForCompare(normalizedStrings, origStringBlock);
+            normStrLen = uprv_strlen(normalizedStrings);
+            if (normStrLen > 0) {
+                uprv_memset(normalizedStrings + normStrLen, 0, currStrSize - normStrLen);
+            }
+        }
+        stringBlockLength -= currStrSize;
+        normalizedStrings += currStrSize;
+        origStringBlock += currStrSize;
+    }
+}
+
+static void
+writeAliasTable(UNewDataMemory *out) {
+    uint32_t i, j;
+    uint32_t uniqueAliasesSize;
+    uint16_t aliasOffset = (uint16_t)(tagBlock.top/sizeof(uint16_t));
+    uint16_t *aliasArrLists = (uint16_t *)uprv_malloc(tagCount * converterCount * sizeof(uint16_t));
+    uint16_t *uniqueAliases = (uint16_t *)uprv_malloc(knownAliasesCount * sizeof(uint16_t));
+    uint16_t *uniqueAliasesToConverter = (uint16_t *)uprv_malloc(knownAliasesCount * sizeof(uint16_t));
+
+    qsort(knownAliases, knownAliasesCount, sizeof(knownAliases[0]), compareAliases);
+    uniqueAliasesSize = resolveAliases(uniqueAliases, uniqueAliasesToConverter, aliasOffset);
+
+    /* Array index starts at 1. aliasLists[0] is the size of the lists section. */
+    aliasListsSize = 0;
+
+    /* write the offsets of all the aliases lists in a 2D array, and create the lists. */
+    for (i = 0; i < tagCount; ++i) {
+        for (j = 0; j < converterCount; ++j) {
+            createOneAliasList(aliasArrLists, i, j, aliasOffset);
+        }
+    }
+
+    /* Write the size of the TOC */
+    if (tableOptions.stringNormalizationType == UCNV_IO_UNNORMALIZED) {
+        udata_write32(out, 8);
+    }
+    else {
+        udata_write32(out, 9);
+    }
+
+    /* Write the sizes of each section */
+    /* All sizes are the number of uint16_t units, not bytes */
+    udata_write32(out, converterCount);
+    udata_write32(out, tagCount);
+    udata_write32(out, uniqueAliasesSize);  /* list of aliases */
+    udata_write32(out, uniqueAliasesSize);  /* The preresolved form of mapping an untagged the alias to a converter */
+    udata_write32(out, tagCount * converterCount);
+    udata_write32(out, aliasListsSize + 1);
+    udata_write32(out, sizeof(tableOptions) / sizeof(uint16_t));
+    udata_write32(out, (tagBlock.top + stringBlock.top) / sizeof(uint16_t));
+    if (tableOptions.stringNormalizationType != UCNV_IO_UNNORMALIZED) {
+        udata_write32(out, (tagBlock.top + stringBlock.top) / sizeof(uint16_t));
+    }
+
+    /* write the table of converters */
+    /* Think of this as the column headers */
+    for(i=0; i<converterCount; ++i) {
+        udata_write16(out, (uint16_t)(converters[i].converter + aliasOffset));
+    }
+
+    /* write the table of tags */
+    /* Think of this as the row headers */
+    for(i=UCNV_NUM_RESERVED_TAGS; i<tagCount; ++i) {
+        udata_write16(out, tags[i].tag);
+    }
+    /* The empty tag is considered the leftover list, and put that at the end of the priority list. */
+    udata_write16(out, tags[EMPTY_TAG_NUM].tag);
+    udata_write16(out, tags[ALL_TAG_NUM].tag);
+
+    /* Write the unique list of aliases */
+    udata_writeBlock(out, uniqueAliases, uniqueAliasesSize * sizeof(uint16_t));
+
+    /* Write the unique list of aliases */
+    udata_writeBlock(out, uniqueAliasesToConverter, uniqueAliasesSize * sizeof(uint16_t));
+
+    /* Write the array to the lists */
+    udata_writeBlock(out, (const void *)(aliasArrLists + (2*converterCount)), (((tagCount - 2) * converterCount) * sizeof(uint16_t)));
+    /* Now write the leftover part of the array for the EMPTY and ALL lists */
+    udata_writeBlock(out, (const void *)aliasArrLists, (2 * converterCount * sizeof(uint16_t)));
+
+    /* Offset the next array to make the index start at 1. */
+    udata_write16(out, 0xDEAD);
+
+    /* Write the lists */
+    udata_writeBlock(out, (const void *)aliasLists, aliasListsSize * sizeof(uint16_t));
+
+    /* Write any options for the alias table. */
+    udata_writeBlock(out, (const void *)&tableOptions, sizeof(tableOptions));
+
+    /* write the tags strings */
+    udata_writeString(out, tagBlock.store, tagBlock.top);
+
+    /* write the aliases strings */
+    udata_writeString(out, stringBlock.store, stringBlock.top);
+
+    /* write the normalized aliases strings */
+    if (tableOptions.stringNormalizationType != UCNV_IO_UNNORMALIZED) {
+        char *normalizedStrings = (char *)uprv_malloc(tagBlock.top + stringBlock.top);
+        createNormalizedAliasStrings(normalizedStrings, tagBlock.store, tagBlock.top);
+        createNormalizedAliasStrings(normalizedStrings + tagBlock.top, stringBlock.store, stringBlock.top);
+
+        /* Write out the complete normalized array. */
+        udata_writeString(out, normalizedStrings, tagBlock.top + stringBlock.top);
+        uprv_free(normalizedStrings);
+    }
+
+    uprv_free(aliasArrLists);
+    uprv_free(uniqueAliases);
+}
+
+static char *
+allocString(StringBlock *block, const char *s, int32_t length) {
+    uint32_t top;
+    char *p;
+
+    if(length<0) {
+        length=(int32_t)uprv_strlen(s);
+    }
+
+    /*
+     * add 1 for the terminating NUL
+     * and round up (+1 &~1)
+     * to keep the addresses on a 16-bit boundary
+     */
+    top=block->top + (uint32_t)((length + 1 + 1) & ~1);
+
+    if(top >= block->max) {
+        fprintf(stderr, "%s:%d: error: out of memory\n", path, lineNum);
+        exit(U_MEMORY_ALLOCATION_ERROR);
+    }
+
+    /* get the pointer and copy the string */
+    p = block->store + block->top;
+    uprv_memcpy(p, s, length);
+    p[length] = 0; /* NUL-terminate it */
+    if((length & 1) == 0) {
+        p[length + 1] = 0; /* set the padding byte */
+    }
+
+    /* check for invariant characters now that we have a NUL-terminated string for easy output */
+    if(!uprv_isInvariantString(p, length)) {
+        fprintf(stderr, "%s:%d: error: the name %s contains not just invariant characters\n", path, lineNum, p);
+        exit(U_INVALID_TABLE_FORMAT);
+    }
+
+    block->top = top;
+    return p;
+}
+
+static int
+compareAliases(const void *alias1, const void *alias2) {
+    /* Names like IBM850 and ibm-850 need to be sorted together */
+    int result = ucnv_compareNames(GET_ALIAS_STR(*(uint16_t*)alias1), GET_ALIAS_STR(*(uint16_t*)alias2));
+    if (!result) {
+        /* Sort the shortest first */
+        return (int)uprv_strlen(GET_ALIAS_STR(*(uint16_t*)alias1)) - (int)uprv_strlen(GET_ALIAS_STR(*(uint16_t*)alias2));
+    }
+    return result;
+}
+
+/*
+ * Hey, Emacs, please set the following:
+ *
+ * Local Variables:
+ * indent-tabs-mode: nil
+ * End:
+ *
+ */
+
Property changes on: icu46/source/tools/gencnval/gencnval.c
___________________________________________________________________
Added: svn:eol-style
   + LF