Update ICU to 54.1 step 1

source/tools/genrb/parse.cpp

 /*
 *******************************************************************************
 *
-*   Copyright (C) 1998-2013, International Business Machines
+*   Copyright (C) 1998-2014, International Business Machines
 *   Corporation and others.  All Rights Reserved.
 *
 *******************************************************************************
 *
 * File parse.cpp
 *
 * Modification History:
 *
 *   Date          Name          Description
 *   05/26/99     stephen       Creation.
 *   02/25/00     weiv          Overhaul to write udata
 *   5/10/01      Ram           removed ustdio dependency
 *   06/10/2001  Dominic Ludlam <dom@recoil.org> Rewritten
 *******************************************************************************
 */
 
-#include "ucol_imp.h"
+// Safer use of UnicodeString.
+#ifndef UNISTR_FROM_CHAR_EXPLICIT
+#   define UNISTR_FROM_CHAR_EXPLICIT explicit
+#endif
+
+// Less important, but still a good idea.
+#ifndef UNISTR_FROM_STRING_EXPLICIT
+#   define UNISTR_FROM_STRING_EXPLICIT explicit
+#endif
+
 #include "parse.h"
 #include "errmsg.h"
 #include "uhash.h"
 #include "cmemory.h"
 #include "cstring.h"
 #include "uinvchar.h"
 #include "read.h"
 #include "ustr.h"
 #include "reslist.h"
 #include "rbt_pars.h"
 #include "genrb.h"
 #include "unicode/ustring.h"
 #include "unicode/uscript.h"
+#include "unicode/utf16.h"
 #include "unicode/putil.h"
+#include "collationbuilder.h"
+#include "collationdata.h"
+#include "collationdatareader.h"
+#include "collationdatawriter.h"
+#include "collationfastlatinbuilder.h"
+#include "collationinfo.h"
+#include "collationroot.h"
+#include "collationruleparser.h"
+#include "collationtailoring.h"
 #include <stdio.h>
 
 /* Number of tokens to read ahead of the current stream position */
 #define MAX_LOOKAHEAD   3
 
 #define CR               0x000D
 #define LF               0x000A
 #define SPACE            0x0020
 #define TAB              0x0009
 #define ESCAPE           0x005C
 #define HASH             0x0023
 #define QUOTE            0x0027
 #define ZERO             0x0030
 #define STARTCOMMAND     0x005B
 #define ENDCOMMAND       0x005D
 #define OPENSQBRACKET    0x005B
 #define CLOSESQBRACKET   0x005D
 
+using icu::LocalPointer;
+using icu::UnicodeString;
+
 struct Lookahead
 {
      enum   ETokenType type;
      struct UString    value;
      struct UString    comment;
      uint32_t          line;
 };
 
 /* keep in sync with token defines in read.h */
 const char *tokenNames[TOK_TOKEN_COUNT] =
@@ skipping 13 matching lines @@
 
 typedef struct {
     struct Lookahead  lookahead[MAX_LOOKAHEAD + 1];
     uint32_t          lookaheadPosition;
     UCHARBUF         *buffer;
     struct SRBRoot *bundle;
     const char     *inputdir;
     uint32_t        inputdirLength;
     const char     *outputdir;
     uint32_t        outputdirLength;
+    const char     *filename;
     UBool           makeBinaryCollation;
+    UBool           omitCollationRules;
 } ParseState;
 
-static UBool gOmitCollationRules  = FALSE;
-
 typedef struct SResource *
 ParseResourceFunction(ParseState* state, char *tag, uint32_t startline, const struct UString* comment, UErrorCode *status);
 
 static struct SResource *parseResource(ParseState* state, char *tag, const struct UString *comment, UErrorCode *status);
 
 /* The nature of the lookahead buffer:
    There are MAX_LOOKAHEAD + 1 slots, used as a circular buffer.  This provides
    MAX_LOOKAHEAD lookahead tokens and a slot for the current token and value.
    When getToken is called, the current pointer is moved to the next slot and the
    old slot is filled with the next token from the reader by calling getNextToken.
@@ skipping 214 matching lines @@
     u_UCharsToChars(tokenValue->fChars, cs, tokenValue->fLength);
 
     expect(state, TOK_CLOSE_BRACE, NULL, NULL, NULL, status);
 
     if (U_FAILURE(*status))
     {
         return NULL;
     }
     uprv_strcat(filename, cs);
 
-    if(gOmitCollationRules) {
+    if(state->omitCollationRules) {
         return res_none();
     }
 
     ucbuf = ucbuf_open(filename, &cp, getShowWarning(),FALSE, status);
 
     if (U_FAILURE(*status)) {
         error(line, "An error occured while opening the input file %s\n", filename);
         return NULL;
     }
 
@@ skipping 280 matching lines @@
         if (U_FAILURE(*status))
         {
             res_close(result);
             return NULL;
         }
     }
 
     return result;
 }
 
-typedef struct{
-    const char* inputDir;
-    const char* outputDir;
-} GenrbData;
+#if !UCONFIG_NO_COLLATION
+
+namespace {
 
 static struct SResource* resLookup(struct SResource* res, const char* key){
     struct SResource *current = NULL;
     struct SResTable *list;
     if (res == res_none()) {
         return NULL;
     }
 
     list = &(res->u.fTable);
 
     current = list->fFirst;
     while (current != NULL) {
         if (uprv_strcmp(((list->fRoot->fKeys) + (current->fKey)), key) == 0) {
             return current;
         }
         current = current->fNext;
     }
     return NULL;
 }
 
-static const UChar* importFromDataFile(void* context, const char* locale, const char* type, int32_t* pLength, UErrorCode* status){
+class GenrbImporter : public icu::CollationRuleParser::Importer {
+public:
+    GenrbImporter(const char *in, const char *out) : inputDir(in), outputDir(out) {}
+    virtual ~GenrbImporter();
+    virtual void getRules(
+            const char *localeID, const char *collationType,
+            UnicodeString &rules,
+            const char *&errorReason, UErrorCode &errorCode);
+
+private:
+    const char *inputDir;
+    const char *outputDir;
+};
+
+GenrbImporter::~GenrbImporter() {}
+
+void
+GenrbImporter::getRules(
+        const char *localeID, const char *collationType,
+        UnicodeString &rules,
+        const char *& /*errorReason*/, UErrorCode &errorCode) {
     struct SRBRoot *data         = NULL;
     UCHARBUF       *ucbuf        = NULL;
-    GenrbData* genrbdata = (GenrbData*) context;
-    int localeLength = strlen(locale);
+    int localeLength = strlen(localeID);
     char* filename = (char*)uprv_malloc(localeLength+5);
     char           *inputDirBuf  = NULL;
     char           *openFileName = NULL;
     const char* cp = "";
-    UChar* urules = NULL;
-    int32_t urulesLength = 0;
     int32_t i = 0;
     int32_t dirlen  = 0;
     int32_t filelen = 0;
     struct SResource* root;
     struct SResource* collations;
     struct SResource* collation;
     struct SResource* sequence;
 
-    memcpy(filename, locale, localeLength);
+    memcpy(filename, localeID, localeLength);
     for(i = 0; i < localeLength; i++){
         if(filename[i] == '-'){
             filename[i] = '_';
         }
     }
     filename[localeLength]   = '.';
     filename[localeLength+1] = 't';
     filename[localeLength+2] = 'x';
     filename[localeLength+3] = 't';
     filename[localeLength+4] = 0;
 
 
-    if (status==NULL || U_FAILURE(*status)) {
-        return NULL;
+    if (U_FAILURE(errorCode)) {
+        return;
     }
     if(filename==NULL){
-        *status=U_ILLEGAL_ARGUMENT_ERROR;
-        return NULL;
+        errorCode=U_ILLEGAL_ARGUMENT_ERROR;
+        return;
     }else{
         filelen = (int32_t)uprv_strlen(filename);
     }
-    if(genrbdata->inputDir == NULL) {
+    if(inputDir == NULL) {
         const char *filenameBegin = uprv_strrchr(filename, U_FILE_SEP_CHAR);
         openFileName = (char *) uprv_malloc(dirlen + filelen + 2);
         openFileName[0] = '\0';
         if (filenameBegin != NULL) {
             /*
              * When a filename ../../../data/root.txt is specified,
              * we presume that the input directory is ../../../data
              * This is very important when the resource file includes
              * another file, like UCARules.txt or thaidict.brk.
              */
             int32_t filenameSize = (int32_t)(filenameBegin - filename + 1);
-            inputDirBuf = uprv_strncpy((char *)uprv_malloc(filenameSize), filename, filenameSize);
+            inputDirBuf = (char *)uprv_malloc(filenameSize);
 
             /* test for NULL */
             if(inputDirBuf == NULL) {
-                *status = U_MEMORY_ALLOCATION_ERROR;
+                errorCode = U_MEMORY_ALLOCATION_ERROR;
                 goto finish;
             }
 
+            uprv_strncpy(inputDirBuf, filename, filenameSize);
             inputDirBuf[filenameSize - 1] = 0;
-            genrbdata->inputDir = inputDirBuf;
-            dirlen  = (int32_t)uprv_strlen(genrbdata->inputDir);
+            inputDir = inputDirBuf;
+            dirlen  = (int32_t)uprv_strlen(inputDir);
         }
     }else{
-        dirlen  = (int32_t)uprv_strlen(genrbdata->inputDir);
+        dirlen  = (int32_t)uprv_strlen(inputDir);
 
-        if(genrbdata->inputDir[dirlen-1] != U_FILE_SEP_CHAR) {
+        if(inputDir[dirlen-1] != U_FILE_SEP_CHAR) {
             openFileName = (char *) uprv_malloc(dirlen + filelen + 2);
 
             /* test for NULL */
             if(openFileName == NULL) {
-                *status = U_MEMORY_ALLOCATION_ERROR;
+                errorCode = U_MEMORY_ALLOCATION_ERROR;
                 goto finish;
             }
 
             openFileName[0] = '\0';
             /*
              * append the input dir to openFileName if the first char in
              * filename is not file seperation char and the last char input directory is  not '.'.
              * This is to support :
              * genrb -s. /home/icu/data
              * genrb -s. icu/data
              * The user cannot mix notations like
              * genrb -s. /icu/data --- the absolute path specified. -s redundant
              * user should use
              * genrb -s. icu/data  --- start from CWD and look in icu/data dir
              */
-            if( (filename[0] != U_FILE_SEP_CHAR) && (genrbdata->inputDir[dirlen-1] !='.')){
-                uprv_strcpy(openFileName, genrbdata->inputDir);
+            if( (filename[0] != U_FILE_SEP_CHAR) && (inputDir[dirlen-1] !='.')){
+                uprv_strcpy(openFileName, inputDir);
                 openFileName[dirlen]     = U_FILE_SEP_CHAR;
             }
             openFileName[dirlen + 1] = '\0';
         } else {
             openFileName = (char *) uprv_malloc(dirlen + filelen + 1);
 
             /* test for NULL */
             if(openFileName == NULL) {
-                *status = U_MEMORY_ALLOCATION_ERROR;
+                errorCode = U_MEMORY_ALLOCATION_ERROR;
                 goto finish;
             }
 
-            uprv_strcpy(openFileName, genrbdata->inputDir);
+            uprv_strcpy(openFileName, inputDir);
 
         }
     }
     uprv_strcat(openFileName, filename);
     /* printf("%s\n", openFileName);  */
-    *status = U_ZERO_ERROR;
-    ucbuf = ucbuf_open(openFileName, &cp,getShowWarning(),TRUE, status);
+    errorCode = U_ZERO_ERROR;
+    ucbuf = ucbuf_open(openFileName, &cp,getShowWarning(),TRUE, &errorCode);
 
-    if(*status == U_FILE_ACCESS_ERROR) {
+    if(errorCode == U_FILE_ACCESS_ERROR) {
 
         fprintf(stderr, "couldn't open file %s\n", openFileName == NULL ? filename : openFileName);
         goto finish;
     }
-    if (ucbuf == NULL || U_FAILURE(*status)) {
-        fprintf(stderr, "An error occured processing file %s. Error: %s\n", openFileName == NULL ? filename : openFileName,u_errorName(*status));
+    if (ucbuf == NULL || U_FAILURE(errorCode)) {
+        fprintf(stderr, "An error occured processing file %s. Error: %s\n", openFileName == NULL ? filename : openFileName,u_errorName(errorCode));
         goto finish;
     }
 
     /* Parse the data into an SRBRoot */
-    data = parse(ucbuf, genrbdata->inputDir, genrbdata->outputDir, FALSE, status);
+    data = parse(ucbuf, inputDir, outputDir, filename, FALSE, FALSE, &errorCode);
+    if (U_FAILURE(errorCode)) {
+        goto finish;
+    }
 
     root = data->fRoot;
     collations = resLookup(root, "collations");
     if (collations != NULL) {
-      collation = resLookup(collations, type);
+      collation = resLookup(collations, collationType);
       if (collation != NULL) {
         sequence = resLookup(collation, "Sequence");
         if (sequence != NULL) {
-          urules = sequence->u.fString.fChars;
-          urulesLength = sequence->u.fString.fLength;
-          *pLength = urulesLength;
+          // No string pointer aliasing so that we need not hold onto the resource bundle.
+          rules.setTo(sequence->u.fString.fChars, sequence->u.fString.fLength);
         }
       }
     }
 
 finish:
     if (inputDirBuf != NULL) {
         uprv_free(inputDirBuf);
     }
 
     if (openFileName != NULL) {
         uprv_free(openFileName);
     }
 
     if(ucbuf) {
         ucbuf_close(ucbuf);
     }
-
-    return urules;
 }
 
 // Quick-and-dirty escaping function.
 // Assumes that we are on an ASCII-based platform.
 static void
 escape(const UChar *s, char *buffer) {
     int32_t length = u_strlen(s);
     int32_t i = 0;
     for (;;) {
         UChar32 c;
         U16_NEXT(s, i, length, c);
         if (c == 0) {
             *buffer = 0;
             return;
         } else if (0x20 <= c && c <= 0x7e) {
             // printable ASCII
             *buffer++ = (char)c;  // assumes ASCII-based platform
         } else {
             buffer += sprintf(buffer, "\\u%04X", (int)c);
         }
     }
 }
 
+}  // namespace
+
+#endif  // !UCONFIG_NO_COLLATION
+
 static struct SResource *
-addCollation(ParseState* state, struct SResource  *result, uint32_t startline, UErrorCode *status)
+addCollation(ParseState* state, struct SResource  *result, const char *collationType,
+             uint32_t startline, UErrorCode *status)
 {
+    // TODO: Use LocalPointer for result, or make caller close it when there is a failure.
     struct SResource  *member = NULL;
     struct UString    *tokenValue;
     struct UString     comment;
     enum   ETokenType  token;
     char               subtag[1024];
+    UnicodeString      rules;
+    UBool              haveRules = FALSE;
     UVersionInfo       version;
     uint32_t           line;
-    GenrbData genrbdata;
+
     /* '{' . (name resource)* '}' */
     version[0]=0; version[1]=0; version[2]=0; version[3]=0;
 
     for (;;)
     {
         ustr_init(&comment);
         token = getToken(state, &tokenValue, &comment, &line, status);
 
         if (token == TOK_CLOSE_BRACE)
         {
-            return result;
+            break;
         }
 
         if (token != TOK_STRING)
         {
             res_close(result);
             *status = U_INVALID_FORMAT_ERROR;
 
             if (token == TOK_EOF)
             {
                 error(startline, "unterminated table");
@@ skipping 14 matching lines @@
             return NULL;
         }
 
         member = parseResource(state, subtag, NULL, status);
 
         if (U_FAILURE(*status))
         {
             res_close(result);
             return NULL;
         }
-
-        if (uprv_strcmp(subtag, "Version") == 0)
+        if (result == NULL)
+        {
+            // Ignore the parsed resources, continue parsing.
+        }
+        else if (uprv_strcmp(subtag, "Version") == 0)
         {
             char     ver[40];
             int32_t length = member->u.fString.fLength;
 
             if (length >= (int32_t) sizeof(ver))
             {
                 length = (int32_t) sizeof(ver) - 1;
             }
 
             u_UCharsToChars(member->u.fString.fChars, ver, length + 1); /* +1 for copying NULL */
             u_versionFromString(version, ver);
 
             table_add(result, member, line, status);
-
-        }
-        else if (uprv_strcmp(subtag, "Override") == 0)
-        {
-            // UBool override = (u_strncmp(member->u.fString.fChars, trueValue, u_strlen(trueValue)) == 0);
-            table_add(result, member, line, status);
-
+            member = NULL;
         }
         else if(uprv_strcmp(subtag, "%%CollationBin")==0)
         {
             /* discard duplicate %%CollationBin if any*/
         }
         else if (uprv_strcmp(subtag, "Sequence") == 0)
         {
-#if UCONFIG_NO_COLLATION || UCONFIG_NO_FILE_IO
-            warning(line, "Not building collation elements because of UCONFIG_NO_COLLATION and/or UCONFIG_NO_FILE_IO, see uconfig.h");
-#else
-            if(state->makeBinaryCollation) {
-
-                /* do the collation elements */
-                int32_t     len   = 0;
-                uint8_t   *data  = NULL;
-                UCollator *coll  = NULL;
-                int32_t reorderCodes[USCRIPT_CODE_LIMIT + (UCOL_REORDER_CODE_LIMIT - UCOL_REORDER_CODE_FIRST)];
-                int32_t reorderCodeCount;
-                int32_t reorderCodeIndex;
-                UParseError parseError;
-
-                genrbdata.inputDir = state->inputdir;
-                genrbdata.outputDir = state->outputdir;
-
-                UErrorCode intStatus = U_ZERO_ERROR;
-                uprv_memset(&parseError, 0, sizeof(parseError));
-                coll = ucol_openRulesForImport(member->u.fString.fChars, member->u.fString.fLength,
-                                               UCOL_OFF, UCOL_DEFAULT_STRENGTH,&parseError, importFromDataFile, &genrbdata, &intStatus);
-
-                if (U_SUCCESS(intStatus) && coll != NULL)
-                {
-                    len = ucol_cloneBinary(coll, NULL, 0, &intStatus);
-                    data = (uint8_t *)uprv_malloc(len);
-                    intStatus = U_ZERO_ERROR;
-                    len = ucol_cloneBinary(coll, data, len, &intStatus);
-
-                    /* tailoring rules version */
-                    /* This is wrong! */
-                    /*coll->dataInfo.dataVersion[1] = version[0];*/
-                    /* Copy tailoring version. Builder version already */
-                    /* set in ucol_openRules */
-                    ((UCATableHeader *)data)->version[1] = version[0];
-                    ((UCATableHeader *)data)->version[2] = version[1];
-                    ((UCATableHeader *)data)->version[3] = version[2];
-
-                    if (U_SUCCESS(intStatus) && data != NULL)
-                    {
-                        struct SResource *collationBin = bin_open(state->bundle, "%%CollationBin", len, data, NULL, NULL, status);
-                        table_add(result, collationBin, line, status);
-                        uprv_free(data);
-
-                        reorderCodeCount = ucol_getReorderCodes(
-                            coll, reorderCodes, USCRIPT_CODE_LIMIT + (UCOL_REORDER_CODE_LIMIT - UCOL_REORDER_CODE_FIRST), &intStatus);
-                        if (U_SUCCESS(intStatus) && reorderCodeCount > 0) {
-                            struct SResource *reorderCodeRes = intvector_open(state->bundle, "%%ReorderCodes", NULL, status);
-                            for (reorderCodeIndex = 0; reorderCodeIndex < reorderCodeCount; reorderCodeIndex++) {
-                                intvector_add(reorderCodeRes, reorderCodes[reorderCodeIndex], status);
-                            }
-                            table_add(result, reorderCodeRes, line, status);
-                        }
-                    }
-                    else
-                    {
-                        warning(line, "could not obtain rules from collator");
-                        if(isStrict()){
-                            *status = U_INVALID_FORMAT_ERROR;
-                            return NULL;
-                        }
-                    }
-
-                    ucol_close(coll);
-                }
-                else
-                {
-                    if(intStatus == U_FILE_ACCESS_ERROR) {
-                        error(startline, "Collation could not be built- U_FILE_ACCESS_ERROR. Make sure ICU's data has been built and is loading properly.");
-                        *status = intStatus;
-                        return NULL;
-                    }
-                    char preBuffer[100], postBuffer[100];
-                    escape(parseError.preContext, preBuffer);
-                    escape(parseError.postContext, postBuffer);
-                    warning(line,
-                            "%%%%CollationBin could not be constructed from CollationElements\n"
-                            "  check context, check that the FractionalUCA.txt UCA version "
-                            "matches the current UCD version\n"
-                            "  UErrorCode=%s  UParseError={ line=%d offset=%d pre=<> post=<> }",
-                            u_errorName(intStatus),
-                            parseError.line,
-                            parseError.offset,
-                            preBuffer,
-                            postBuffer);
-                    if(isStrict()){
-                        *status = intStatus;
-                        return NULL;
-                    }
-                }
-            } else {
-                if(isVerbose()) {
-                    printf("Not building Collation binary\n");
-                }
-            }
-#endif
+            rules.setTo(member->u.fString.fChars, member->u.fString.fLength);
+            haveRules = TRUE;
+            // Defer building the collator until we have seen
+            // all sub-elements of the collation table, including the Version.
             /* in order to achieve smaller data files, we can direct genrb */
             /* to omit collation rules */
-            if(gOmitCollationRules) {
-                bundle_closeString(state->bundle, member);
-            } else {
+            if(!state->omitCollationRules) {
                 table_add(result, member, line, status);
+                member = NULL;
             }
         }
+        else  // Just copy non-special items.
+        {
+            table_add(result, member, line, status);
+            member = NULL;
+        }
+        res_close(member);  // TODO: use LocalPointer
         if (U_FAILURE(*status))
         {
             res_close(result);
             return NULL;
         }
     }
 
-    // Reached the end without a TOK_CLOSE_BRACE.  Should be an error.
-    *status = U_INTERNAL_PROGRAM_ERROR;
-    return NULL;
+    if (!haveRules) { return result; }
+
+#if UCONFIG_NO_COLLATION || UCONFIG_NO_FILE_IO
+    warning(line, "Not building collation elements because of UCONFIG_NO_COLLATION and/or UCONFIG_NO_FILE_IO, see uconfig.h");
+    (void)collationType;
+#else
+    // CLDR ticket #3949, ICU ticket #8082:
+    // Do not build collation binary data for for-import-only "private" collation rule strings.
+    if (uprv_strncmp(collationType, "private-", 8) == 0) {
+        if(isVerbose()) {
+            printf("Not building %s~%s collation binary\n", state->filename, collationType);
+        }
+        return result;
+    }
+
+    if(!state->makeBinaryCollation) {
+        if(isVerbose()) {
+            printf("Not building %s~%s collation binary\n", state->filename, collationType);
+        }
+        return result;
+    }
+    UErrorCode intStatus = U_ZERO_ERROR;
+    UParseError parseError;
+    uprv_memset(&parseError, 0, sizeof(parseError));
+    GenrbImporter importer(state->inputdir, state->outputdir);
+    const icu::CollationTailoring *base = icu::CollationRoot::getRoot(intStatus);
+    if(U_FAILURE(intStatus)) {
+        error(line, "failed to load root collator (ucadata.icu) - %s", u_errorName(intStatus));
+        res_close(result);
+        return NULL;  // TODO: use LocalUResourceBundlePointer for result
+    }
+    icu::CollationBuilder builder(base, intStatus);
+    if(uprv_strncmp(collationType, "search", 6) == 0) {
+        builder.disableFastLatin();  // build fast-Latin table unless search collator
+    }
+    LocalPointer<icu::CollationTailoring> t(
+            builder.parseAndBuild(rules, version, &importer, &parseError, intStatus));
+    if(U_FAILURE(intStatus)) {
+        const char *reason = builder.getErrorReason();
+        if(reason == NULL) { reason = ""; }
+        error(line, "CollationBuilder failed at %s~%s/Sequence rule offset %ld: %s  %s",
+                state->filename, collationType,
+                (long)parseError.offset, u_errorName(intStatus), reason);
+        if(parseError.preContext[0] != 0 || parseError.postContext[0] != 0) {
+            // Print pre- and post-context.
+            char preBuffer[100], postBuffer[100];
+            escape(parseError.preContext, preBuffer);
+            escape(parseError.postContext, postBuffer);
+            error(line, "  error context: \"...%s\" ! \"%s...\"", preBuffer, postBuffer);
+        }
+        if(isStrict()) {
+            *status = intStatus;
+            res_close(result);
+            return NULL;
+        }
+    }
+    icu::LocalMemory<uint8_t> buffer;
+    int32_t capacity = 100000;
+    uint8_t *dest = buffer.allocateInsteadAndCopy(capacity);
+    if(dest == NULL) {
+        fprintf(stderr, "memory allocation (%ld bytes) for file contents failed\n",
+                (long)capacity);
+        *status = U_MEMORY_ALLOCATION_ERROR;
+        res_close(result);
+        return NULL;
+    }
+    int32_t indexes[icu::CollationDataReader::IX_TOTAL_SIZE + 1];
+    int32_t totalSize = icu::CollationDataWriter::writeTailoring(
+            *t, *t->settings, indexes, dest, capacity, intStatus);
+    if(intStatus == U_BUFFER_OVERFLOW_ERROR) {
+        intStatus = U_ZERO_ERROR;
+        capacity = totalSize;
+        dest = buffer.allocateInsteadAndCopy(capacity);
+        if(dest == NULL) {
+            fprintf(stderr, "memory allocation (%ld bytes) for file contents failed\n",
+                    (long)capacity);
+            *status = U_MEMORY_ALLOCATION_ERROR;
+            res_close(result);
+            return NULL;
+        }
+        totalSize = icu::CollationDataWriter::writeTailoring(
+                *t, *t->settings, indexes, dest, capacity, intStatus);
+    }
+    if(U_FAILURE(intStatus)) {
+        fprintf(stderr, "CollationDataWriter::writeTailoring() failed: %s\n",
+                u_errorName(intStatus));
+        res_close(result);
+        return NULL;
+    }
+    if(isVerbose()) {
+        printf("%s~%s collation tailoring part sizes:\n", state->filename, collationType);
+        icu::CollationInfo::printSizes(totalSize, indexes);
+    }
+    struct SResource *collationBin = bin_open(state->bundle, "%%CollationBin", totalSize, dest, NULL, NULL, status);
+    table_add(result, collationBin, line, status);
+    if (U_FAILURE(*status)) {
+        res_close(result);
+        return NULL;
+    }
+#endif
+    return result;
+}
+
+static UBool
+keepCollationType(const char * /*type*/) {
+    return TRUE;
 }
 
 static struct SResource *
 parseCollationElements(ParseState* state, char *tag, uint32_t startline, UBool newCollation, UErrorCode *status)
 {
     struct SResource  *result = NULL;
     struct SResource  *member = NULL;
     struct SResource  *collationRes = NULL;
     struct UString    *tokenValue;
     struct UString     comment;
     enum   ETokenType  token;
     char               subtag[1024], typeKeyword[1024];
     uint32_t           line;
 
     result = table_open(state->bundle, tag, NULL, status);
 
     if (result == NULL || U_FAILURE(*status))
     {
         return NULL;
     }
     if(isVerbose()){
         printf(" collation elements %s at line %i \n",  (tag == NULL) ? "(null)" : tag, (int)startline);
     }
     if(!newCollation) {
-        return addCollation(state, result, startline, status);
+        return addCollation(state, result, "(no type)", startline, status);
     }
     else {
         for(;;) {
             ustr_init(&comment);
             token = getToken(state, &tokenValue, &comment, &line, status);
 
             if (token == TOK_CLOSE_BRACE)
             {
                 return result;
             }
@@ skipping 36 matching lines @@
                 table_add(result, member, line, status);
             }
             else
             {
                 token = peekToken(state, 0, &tokenValue, &line, &comment, status);
                 /* this probably needs to be refactored or recursively use the parser */
                 /* first we assume that our collation table won't have the explicit type */
                 /* then, we cannot handle aliases */
                 if(token == TOK_OPEN_BRACE) {
                     token = getToken(state, &tokenValue, &comment, &line, status);
-                    collationRes = table_open(state->bundle, subtag, NULL, status);
-                    collationRes = addCollation(state, collationRes, startline, status); /* need to parse the collation data regardless */
-                    if (gIncludeUnihanColl || uprv_strcmp(subtag, "unihan") != 0) {
+                    if (keepCollationType(subtag)) {
+                        collationRes = table_open(state->bundle, subtag, NULL, status);
+                    } else {
+                        collationRes = NULL;
+                    }
+                    // need to parse the collation data regardless
+                    collationRes = addCollation(state, collationRes, subtag, startline, status);
+                    if (collationRes != NULL) {
                         table_add(result, collationRes, startline, status);
                     }
                 } else if(token == TOK_COLON) { /* right now, we'll just try to see if we have aliases */
                     /* we could have a table too */
                     token = peekToken(state, 1, &tokenValue, &line, &comment, status);
                     u_UCharsToChars(tokenValue->fChars, typeKeyword, u_strlen(tokenValue->fChars) + 1);
                     if(uprv_strcmp(typeKeyword, "alias") == 0) {
                         member = parseResource(state, subtag, NULL, status);
                         if (U_FAILURE(*status))
                         {
@@ skipping 671 matching lines @@
 U_STRING_DECL(k_type_include,   "include",   7);
 
 /* Various non-standard processing plugins that create one or more special resources. */
 U_STRING_DECL(k_type_plugin_uca_rules,      "process(uca_rules)",        18);
 U_STRING_DECL(k_type_plugin_collation,      "process(collation)",        18);
 U_STRING_DECL(k_type_plugin_transliterator, "process(transliterator)",   23);
 U_STRING_DECL(k_type_plugin_dependency,     "process(dependency)",       19);
 
 typedef enum EResourceType
 {
-    RT_UNKNOWN,
-    RT_STRING,
-    RT_BINARY,
-    RT_TABLE,
-    RT_TABLE_NO_FALLBACK,
-    RT_INTEGER,
-    RT_ARRAY,
-    RT_ALIAS,
-    RT_INTVECTOR,
-    RT_IMPORT,
-    RT_INCLUDE,
-    RT_PROCESS_UCA_RULES,
-    RT_PROCESS_COLLATION,
-    RT_PROCESS_TRANSLITERATOR,
-    RT_PROCESS_DEPENDENCY,
-    RT_RESERVED
+    RESTYPE_UNKNOWN,
+    RESTYPE_STRING,
+    RESTYPE_BINARY,
+    RESTYPE_TABLE,
+    RESTYPE_TABLE_NO_FALLBACK,
+    RESTYPE_INTEGER,
+    RESTYPE_ARRAY,
+    RESTYPE_ALIAS,
+    RESTYPE_INTVECTOR,
+    RESTYPE_IMPORT,
+    RESTYPE_INCLUDE,
+    RESTYPE_PROCESS_UCA_RULES,
+    RESTYPE_PROCESS_COLLATION,
+    RESTYPE_PROCESS_TRANSLITERATOR,
+    RESTYPE_PROCESS_DEPENDENCY,
+    RESTYPE_RESERVED
 } EResourceType;
 
 static struct {
     const char *nameChars;   /* only used for debugging */
     const UChar *nameUChars;
     ParseResourceFunction *parseFunction;
 } gResourceTypes[] = {
     {"Unknown", NULL, NULL},
     {"string", k_type_string, parseString},
     {"binary", k_type_binary, parseBinary},
     {"table", k_type_table, parseTable},
     {"table(nofallback)", k_type_table_no_fallback, NULL}, /* parseFunction will never be called */
     {"integer", k_type_integer, parseInteger},
     {"array", k_type_array, parseArray},
     {"alias", k_type_alias, parseAlias},
     {"intvector", k_type_intvector, parseIntVector},
     {"import", k_type_import, parseImport},
     {"include", k_type_include, parseInclude},
     {"process(uca_rules)", k_type_plugin_uca_rules, parseUCARules},
     {"process(collation)", k_type_plugin_collation, NULL /* not implemented yet */},
     {"process(transliterator)", k_type_plugin_transliterator, parseTransliterator},
     {"process(dependency)", k_type_plugin_dependency, parseDependency},
     {"reserved", NULL, NULL}
 };
 
-void initParser(UBool omitCollationRules)
+void initParser()
 {
     U_STRING_INIT(k_type_string,    "string",    6);
     U_STRING_INIT(k_type_binary,    "binary",    6);
     U_STRING_INIT(k_type_bin,       "bin",       3);
     U_STRING_INIT(k_type_table,     "table",     5);
     U_STRING_INIT(k_type_table_no_fallback,     "table(nofallback)",         17);
     U_STRING_INIT(k_type_int,       "int",       3);
     U_STRING_INIT(k_type_integer,   "integer",   7);
     U_STRING_INIT(k_type_array,     "array",     5);
     U_STRING_INIT(k_type_alias,     "alias",     5);
     U_STRING_INIT(k_type_intvector, "intvector", 9);
     U_STRING_INIT(k_type_import,    "import",    6);
     U_STRING_INIT(k_type_include,   "include",   7);
 
     U_STRING_INIT(k_type_plugin_uca_rules,      "process(uca_rules)",        18);
     U_STRING_INIT(k_type_plugin_collation,      "process(collation)",        18);
     U_STRING_INIT(k_type_plugin_transliterator, "process(transliterator)",   23);
     U_STRING_INIT(k_type_plugin_dependency,     "process(dependency)",       19);
-
-    gOmitCollationRules = omitCollationRules;
 }
 
 static inline UBool isTable(enum EResourceType type) {
-    return (UBool)(type==RT_TABLE || type==RT_TABLE_NO_FALLBACK);
+    return (UBool)(type==RESTYPE_TABLE || type==RESTYPE_TABLE_NO_FALLBACK);
 }
 
 static enum EResourceType
 parseResourceType(ParseState* state, UErrorCode *status)
 {
     struct UString        *tokenValue;
     struct UString        comment;
-    enum   EResourceType  result = RT_UNKNOWN;
+    enum   EResourceType  result = RESTYPE_UNKNOWN;
     uint32_t              line=0;
     ustr_init(&comment);
     expect(state, TOK_STRING, &tokenValue, &comment, &line, status);
 
     if (U_FAILURE(*status))
     {
-        return RT_UNKNOWN;
+        return RESTYPE_UNKNOWN;
     }
 
     *status = U_ZERO_ERROR;
 
     /* Search for normal types */
-    result=RT_UNKNOWN;
-    while ((result=(EResourceType)(result+1)) < RT_RESERVED) {
+    result=RESTYPE_UNKNOWN;
+    while ((result=(EResourceType)(result+1)) < RESTYPE_RESERVED) {
         if (u_strcmp(tokenValue->fChars, gResourceTypes[result].nameUChars) == 0) {
             break;
         }
     }
     /* Now search for the aliases */
     if (u_strcmp(tokenValue->fChars, k_type_int) == 0) {
-        result = RT_INTEGER;
+        result = RESTYPE_INTEGER;
     }
     else if (u_strcmp(tokenValue->fChars, k_type_bin) == 0) {
-        result = RT_BINARY;
+        result = RESTYPE_BINARY;
     }
-    else if (result == RT_RESERVED) {
+    else if (result == RESTYPE_RESERVED) {
         char tokenBuffer[1024];
         u_austrncpy(tokenBuffer, tokenValue->fChars, sizeof(tokenBuffer));
         tokenBuffer[sizeof(tokenBuffer) - 1] = 0;
         *status = U_INVALID_FORMAT_ERROR;
         error(line, "unknown resource type '%s'", tokenBuffer);
     }
 
     return result;
 }
 
 /* parse a non-top-level resource */
 static struct SResource *
 parseResource(ParseState* state, char *tag, const struct UString *comment, UErrorCode *status)
 {
     enum   ETokenType      token;
-    enum   EResourceType  resType = RT_UNKNOWN;
+    enum   EResourceType  resType = RESTYPE_UNKNOWN;
     ParseResourceFunction *parseFunction = NULL;
     struct UString        *tokenValue;
     uint32_t                 startline;
     uint32_t                 line;
 
 
     token = getToken(state, &tokenValue, NULL, &startline, status);
 
     if(isVerbose()){
         printf(" resource %s at line %i \n",  (tag == NULL) ? "(null)" : tag, (int)startline);
@@ skipping 28 matching lines @@
     case TOK_OPEN_BRACE:
         break;
 
     default:
         *status = U_INVALID_FORMAT_ERROR;
         error(startline, "syntax error while reading a resource, expected '{' or ':'");
         return NULL;
     }
 
 
-    if (resType == RT_UNKNOWN)
+    if (resType == RESTYPE_UNKNOWN)
     {
         /* No explicit type, so try to work it out.  At this point, we've read the first '{'.
         We could have any of the following:
         { {         => array (nested)
         { :/}       => array
         { string ,  => string array
 
         { string {  => table
 
         { string :/{    => table
         { string }      => string
         */
 
         token = peekToken(state, 0, NULL, &line, NULL,status);
 
         if (U_FAILURE(*status))
         {
             return NULL;
         }
 
         if (token == TOK_OPEN_BRACE || token == TOK_COLON ||token ==TOK_CLOSE_BRACE )
         {
-            resType = RT_ARRAY;
+            resType = RESTYPE_ARRAY;
         }
         else if (token == TOK_STRING)
         {
             token = peekToken(state, 1, NULL, &line, NULL, status);
 
             if (U_FAILURE(*status))
             {
                 return NULL;
             }
 
             switch (token)
             {
-            case TOK_COMMA:         resType = RT_ARRAY;  break;
-            case TOK_OPEN_BRACE:    resType = RT_TABLE;  break;
-            case TOK_CLOSE_BRACE:   resType = RT_STRING; break;
-            case TOK_COLON:         resType = RT_TABLE;  break;
+            case TOK_COMMA:         resType = RESTYPE_ARRAY;  break;
+            case TOK_OPEN_BRACE:    resType = RESTYPE_TABLE;  break;
+            case TOK_CLOSE_BRACE:   resType = RESTYPE_STRING; break;
+            case TOK_COLON:         resType = RESTYPE_TABLE;  break;
             default:
                 *status = U_INVALID_FORMAT_ERROR;
                 error(line, "Unexpected token after string, expected ',', '{' or '}'");
                 return NULL;
             }
         }
         else
         {
             *status = U_INVALID_FORMAT_ERROR;
             error(line, "Unexpected token after '{'");
             return NULL;
         }
 
         /* printf("Type guessed as %s\n", resourceNames[resType]); */
-    } else if(resType == RT_TABLE_NO_FALLBACK) {
+    } else if(resType == RESTYPE_TABLE_NO_FALLBACK) {
         *status = U_INVALID_FORMAT_ERROR;
         error(startline, "error: %s resource type not valid except on top bundle level", gResourceTypes[resType].nameChars);
         return NULL;
     }
 
 
     /* We should now know what we need to parse next, so call the appropriate parser
     function and return. */
     parseFunction = gResourceTypes[resType].parseFunction;
     if (parseFunction != NULL) {
         return parseFunction(state, tag, startline, comment, status);
     }
     else {
         *status = U_INTERNAL_PROGRAM_ERROR;
         error(startline, "internal error: %s resource type found and not handled", gResourceTypes[resType].nameChars);
     }
 
     return NULL;
 }
 
 /* parse the top-level resource */
 struct SRBRoot *
-parse(UCHARBUF *buf, const char *inputDir, const char *outputDir, UBool makeBinaryCollation,
-      UErrorCode *status)
+parse(UCHARBUF *buf, const char *inputDir, const char *outputDir, const char *filename,
+      UBool makeBinaryCollation, UBool omitCollationRules, UErrorCode *status)
 {
     struct UString    *tokenValue;
     struct UString    comment;
     uint32_t           line;
     enum EResourceType bundleType;
     enum ETokenType    token;
     ParseState state;
     uint32_t i;
 
 
     for (i = 0; i < MAX_LOOKAHEAD + 1; i++)
     {
         ustr_init(&state.lookahead[i].value);
         ustr_init(&state.lookahead[i].comment);
     }
 
     initLookahead(&state, buf, status);
 
     state.inputdir       = inputDir;
     state.inputdirLength = (state.inputdir != NULL) ? (uint32_t)uprv_strlen(state.inputdir) : 0;
     state.outputdir       = outputDir;
     state.outputdirLength = (state.outputdir != NULL) ? (uint32_t)uprv_strlen(state.outputdir) : 0;
+    state.filename = filename;
     state.makeBinaryCollation = makeBinaryCollation;
+    state.omitCollationRules = omitCollationRules;
 
     ustr_init(&comment);
     expect(&state, TOK_STRING, &tokenValue, &comment, NULL, status);
 
     state.bundle = bundle_open(&comment, FALSE, status);
 
     if (state.bundle == NULL || U_FAILURE(*status))
     {
         return NULL;
     }
@@ skipping 16 matching lines @@
             *status=U_PARSE_ERROR;
              error(line, "parse error. Stopped parsing with %s", u_errorName(*status));
         }
     }
     else
     {
         /* not a colon */
         if(token==TOK_OPEN_BRACE)
         {
             *status=U_ZERO_ERROR;
-            bundleType=RT_TABLE;
+            bundleType=RESTYPE_TABLE;
         }
         else
         {
             /* neither colon nor open brace */
             *status=U_PARSE_ERROR;
-            bundleType=RT_UNKNOWN;
+            bundleType=RESTYPE_UNKNOWN;
             error(line, "parse error, did not find open-brace '{' or colon ':', stopped with %s", u_errorName(*status));
         }
     }
 
     if (U_FAILURE(*status))
     {
         bundle_close(state.bundle, status);
         return NULL;
     }
 
-    if(bundleType==RT_TABLE_NO_FALLBACK) {
+    if(bundleType==RESTYPE_TABLE_NO_FALLBACK) {
         /*
          * Parse a top-level table with the table(nofallback) declaration.
          * This is the same as a regular table, but also sets the
          * URES_ATT_NO_FALLBACK flag in indexes[URES_INDEX_ATTRIBUTES] .
          */
         state.bundle->noFallback=TRUE;
     }
     /* top-level tables need not handle special table names like "collations" */
     realParseTable(&state, state.bundle->fRoot, NULL, line, status);
     if(dependencyArray!=NULL){
@@ skipping 13 matching lines @@
         if(isStrict()){
             *status = U_INVALID_FORMAT_ERROR;
             return NULL;
         }
     }
 
     cleanupLookahead(&state);
     ustr_deinit(&comment);
     return state.bundle;
 }