Check in the pristine copy of ICU 4.6...

icu46/source/test/intltest/wbnf.cpp

+/*
+ ******************************************************************************
+ * Copyright (C) 2005-2007, International Business Machines Corporation and   *
+ * others. All Rights Reserved.                                               *
+ ******************************************************************************
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <time.h>
+
+#include "wbnf.h"
+    
+// Most of this code is meant to test the test code. It's a self test.
+// Normally this isn't run.
+#define TEST_WBNF_TEST 0
+
+///////////////////////////////////////////////////////////
+//
+// Constants and the most basic helper classes
+//
+    
+static const char DIGIT_CHAR[] = "0123456789";
+static const char WHITE_SPACE[] = {'\t', ' ', '\r', '\n', 0};
+static const char ALPHABET[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
+static const char SPECIAL[] = "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~";
+
+static inline UBool isInList(const char c /*in*/, const char list[] /*in*/){
+    const char * p = list;
+    for (;*p != 0 && *p != c; p++);
+    return *p?TRUE:FALSE;
+}
+static inline UBool isDigit(char c) {return isInList(c, DIGIT_CHAR);}
+static inline UBool isWhiteSpace(char c) {return isInList(c, WHITE_SPACE);}
+static inline UBool isAlphabet(char c) {return isInList(c, ALPHABET);}
+static inline UBool isSpecialAsciiChar(char c) {return isInList(c,SPECIAL);}
+
+    
+
+///////////////////////////////////////////////////////////
+//
+// Helper classes
+//
+    
+class Buffer_byte{
+// Utility class, can be treated as an auto expanded array. no boundary check.
+
+    typedef char byte;
+    byte * start;
+    byte * current;
+    int buffer_size; // size unit is byte
+public:
+    inline int content_size(){return current - start;} // size unit is byte
+
+private:
+    inline void expand(int add_size = 100){ // size unit is byte
+        int new_size = buffer_size + add_size;
+
+        int cs_snap = content_size();         
+        start = (byte *) realloc(start, new_size);   // may change the value of start
+        current = start + cs_snap;
+
+        memset(current, 0, add_size);
+        buffer_size = new_size;
+    }
+
+    inline void expand_to(int size){
+        int r = size - buffer_size;
+        if (r > 0) {
+            expand(r);  // simply expand, no block alignment
+        }
+    }
+    Buffer_byte(const Buffer_byte &);
+    Buffer_byte & operator = (const Buffer_byte &);
+public:
+    Buffer_byte():start(NULL),current(start),buffer_size(0){
+        expand();
+    }
+    ~Buffer_byte(){
+        free(start);
+    }
+
+    inline void reset(){
+        start != NULL ? memset(start, 0, buffer_size) : 0;
+        current = start;
+    }
+
+    // Using memory copy method to append a C array to buffer, 
+    inline void append(const void * c, int size){ // size unit is byte
+        expand_to(content_size() + size) ;
+        memcpy(current, c, size);
+        current = current + size;
+    }
+    
+    byte * buffer(){
+        return start;
+    }
+};
+    
+/*
+  The class(es) try to work as bulid-in array, so it overloads these two operators
+    operator type *();
+    type & operator[];
+  The first is used to auto type convert, the latter is used to select member.
+
+  A small trick is the class does not overload the address-of operator. This 
+  behavior is different from bulid-in array, but it give us the opportunity 
+  to get the address of the class itself.
+*/
+//template<typename type>
+//    class BUFFER{
+//       typedef BUFFER name;
+#define BUFFER(type, name)\
+    class name {\
+    private:\
+       Buffer_byte buf;\
+    public:\
+        name & reset() {buf.reset(); return *this;}\
+        name & append(type c) {buf.append(&c, sizeof(type)); return *this;}\
+        name & append_array(const type * p, int size) {buf.append(p, sizeof(type)*size); return *this;}\
+        type & operator [] (int i) { return ((type *) buf.buffer())[i];}\
+        operator type *(){return (type *) buf.buffer();} \
+        int content_size(){return buf.content_size() / sizeof(type);}\
+    }
+    
+
+class Pick{
+/* The Pick is the basic language generator element*/
+public:
+    // generate a string accroding the syntax 
+    // Return a null-terminated c-string. The buffer is owned by callee.
+    virtual const char* next() = 0; 
+    virtual ~Pick(){};
+};
+
+//typedef BUFFER<char> Buffer_char;
+//typedef BUFFER<int> Buffer_int;
+//typedef BUFFER<Pick *> Buffer_pPick;
+BUFFER(char, Buffer_char);
+BUFFER(int, Buffer_int);
+BUFFER(Pick *, Buffer_pPick);
+
+class SymbolTable{
+/* Helper class.
+* It's a mapping table between 'variable name' and its 'active Pick object'
+*/
+private:
+    Buffer_char  name_buffer;   // var names storage space
+
+    Buffer_int   names;         // points to name (offset in name_buffer)
+    Buffer_pPick refs;          // points to Pick
+
+    int get_index(const char *const var_name){
+        int len = names.content_size();
+        for (int i=0; i< len; i++){
+            if (strcmp(var_name, name_buffer + names[i]) == 0){
+                return i;
+            }
+        }
+        return -1;
+    }
+
+public:
+    enum RESULT {EMPTY, NO_VAR, NO_REF, HAS_REF};
+
+    RESULT find(const char *const var_name /*[in] c-string*/, Pick * * ref = NULL /*[out] Pick* */){
+        if (!var_name) return EMPTY; // NULL name
+
+        int i = get_index(var_name);
+        if (i == -1){
+            return NO_VAR;   // new name
+        }
+        if (!refs[i]){  // exist name, no ref
+            return NO_REF;
+        } else {
+            if (ref) {
+                *ref = refs[i];
+            }
+            return HAS_REF;   // exist name, has ref
+        }
+    }
+
+    void put(const char *const var_name, Pick *const var_ref = NULL){
+        int i = get_index(var_name);
+        switch(find(var_name)){
+            case EMPTY:    // NULL name
+                break;
+            case NO_VAR:    // new name
+                int offset;
+                offset = name_buffer.content_size();
+                name_buffer.append_array(var_name, strlen(var_name) + 1);
+                names.append(offset);
+                refs.append(var_ref);
+                break;
+            case NO_REF:    // exist name, no ref
+                refs[i] = var_ref;    // link definition with variable
+                break;
+            case HAS_REF:    // exist name, has ref
+                if (var_ref){
+                    refs[i] = var_ref;
+                }
+                break;
+            default:
+                ; // ASSERT(FALSE);
+        }
+        return;
+    }
+
+    UBool is_complete(){
+        int n = names.content_size();
+        for (int i=0; i<n; ++i){
+            if (refs[i] == NULL){
+                return FALSE;
+            }
+        }
+        return TRUE;
+    }
+
+    void reset(){
+        names.reset();
+        name_buffer.reset();
+
+        // release memory here
+        int s = refs.content_size();
+        for (int i=0; i < s; i++){
+            delete refs[i]; // TOFIX: point alias/recursion problem
+        }
+        refs.reset();
+    }
+
+    ~SymbolTable(){
+        reset();
+    }
+};
+
+
+/*
+// Document of class Escaper
+//
+// ATTENTION: 
+// From http://icu-project.org/userguide/Collate_Customization.html.
+// We get the precedence of escape/quote operations
+//
+//     (highest) 1. backslash               \
+//               2. two single quotes       ''
+//               3. quoting                 ' '
+//
+// ICU Collation should accept following as the same string.
+//
+// 1)  'ab'c        _
+// 2)  a\bc          \
+// 3)  a'b'\c        |- They are equal.
+// 4)  abc          _/
+//
+// From "two single quotes", we have following deductions
+//    D1. empty quoting is illgal. (obviously)
+//    D2. no contact operation between two quotings   
+//              '.''.'      is not ..   it is .'.
+//    D3. "two single quotes" cannot contact two quoting simultaneously
+//              '..''''.'   is not ..'. it is ..''.
+//       NOTICE:
+//        "two single quotes" can contact before one quoting
+//              '''.'       is '.
+//        "two single quotes" can literally contact after one quoting
+//        But, from syntax, it's one quoting including a "two single quotes"
+//              '.'''       is .'
+//    D4. "two single quotes" cannot solely be included in quoting
+//              ''''        is not '    it is ''
+//       NOTICE:  These are legal
+//              '.''.'      is .'.
+//              '.'''       is .'
+//
+//                 dicision
+//                    /\
+//                   /__\
+//      output buffer    input buffer
+// 
+// To make our dicision (within an atom operation) without caring input and output buffer,
+// following calling pattern (within an atom operation) shall be avoided
+//
+//    P1 open_quoting()  then close_quoting()    (direct violation)   D1
+//    P2 close_quoting() then open_quoting()     (direct violation)   D2
+//    P3 empty open_quoting()                    (indirect violation) D1, D4
+//    P4 empty close_quoting()                   (indirect violation) D2, D3
+//    P5 open_quoting()  then two single quotes  (indirect violation) D4
+//    P6 close_quoting() then two single quotes  (indirect violation) D3
+//
+// two single quotes escaping will not open_ or close_ quoting()
+// The choice will not lose some quoing forms.
+//
+// For open_quoting(), 
+// we may get this form quoting     '''         P5
+// It may raise a bug               ''''x
+// If we expect
+//      '''.'       let the next char open the quoting
+//      '.''.'      the quoting is already opened by preceding char
+//
+// For close_quoting()
+// we will get this form quoting    '.'''       P6
+// It may raise a bug               '.''''.'
+// If we expect          
+//      '.'''\.     let the next char close the quoting
+//      '.''''.'    the expectation is wrong!  using  '.'\''.' instead
+//
+// It's a hard work to re-adjust generation opportunity for various escaping form.
+// We just simply ignore it.
+*/
+class Escaper{
+public:
+    enum CHOICE {YES, NO, RAND};
+    enum ESCAPE_FORM {BSLASH_ONLY, QUOTE_ONLY, QUOTE_AND_BSLAH, RAND_ESC};
+private:
+    class Bool{ // A wrapper class for CHOICE, to auto adapter UBool class
+        private:
+            const CHOICE tag;
+        public:
+            Bool(CHOICE flag=RAND):tag(flag){}
+            operator UBool() {   // conversion operator
+                return tag == RAND ? rand()%2 : tag == YES;
+                //if (tag == RAND){
+                //    return rand()%2 == 1;
+                //} else {
+                //    return tag == YES ? TRUE : FALSE;
+                //}
+            }
+    };
+public:
+    Escaper(CHOICE escapeLiteral = RAND,
+        CHOICE twoQuotesEscape = RAND,
+        ESCAPE_FORM escapeForm = RAND_ESC):
+        escape_form(escapeForm),
+        escape_literal(escapeLiteral),
+        two_quotes_escape(twoQuotesEscape),
+        is_quoting(FALSE){}
+private:
+    Buffer_char str;
+    ESCAPE_FORM escape_form;
+    Bool escape_literal;
+    Bool two_quotes_escape;
+    UBool quote_escape;
+    UBool bslash_escape;
+    UBool is_quoting;
+
+    void set_options(){
+        ESCAPE_FORM t = escape_form == RAND_ESC ? (ESCAPE_FORM) (rand()%3) : escape_form;
+        switch (t){
+                case BSLASH_ONLY :
+                    bslash_escape = TRUE; quote_escape = FALSE; break;
+                case QUOTE_ONLY:
+                    bslash_escape = FALSE;quote_escape = TRUE;  break;
+                case QUOTE_AND_BSLAH:
+                    bslash_escape = TRUE; quote_escape = TRUE;  break;
+                default:
+                    ;// error
+        }
+    }
+
+    void reset(){
+        str.reset();
+        is_quoting = FALSE;
+    }
+
+    inline void open_quoting(){ 
+        if(is_quoting){
+            // do nothing
+        } else {
+            str.append('\'');
+            is_quoting = TRUE;
+        }
+    }
+    inline void close_quoting(){
+        if(is_quoting){
+            str.append('\'');
+            is_quoting = FALSE;
+        } else {
+            // do nothing
+        }
+    }
+
+    // str  [in]    null-terminated c-string
+    void append(const char * strToAppend){
+        for(;*strToAppend != 0; strToAppend++){
+            append(*strToAppend);
+        }
+    }
+
+    inline void append(const char c){
+        set_options();
+
+        if (c == '\\'){
+            quote_escape ? open_quoting() : close_quoting();
+            //bslash_escape always true here
+            str.append('\\');
+            str.append('\\');
+        } else if (c == '\''){
+            if (two_quotes_escape){     // quoted using two single quotes
+                // See documents in anonymous.design
+                str.append('\'');
+                str.append('\'');
+            } else{
+                quote_escape ? open_quoting() : close_quoting();
+                //bslash_escape always true here
+                str.append('\\');
+                str.append('\'');
+            }
+        } else if (isSpecialAsciiChar(c) || isWhiteSpace(c)){
+            quote_escape  ? open_quoting()   : close_quoting();
+            if (bslash_escape) str.append('\\');
+            str.append(c);
+        } else { //if (isAlphabet(c) || isDigit(c) || TRUE){ // treat others as literal
+            if (escape_literal){
+                quote_escape  ? open_quoting()   : close_quoting();
+                if (bslash_escape)  str.append('\\');
+                str.append(c);
+            } else {
+                close_quoting();
+                str.append(c);
+            }
+        }
+    }
+
+public:
+    // Return a null-terminate c-string. The buffer is owned by callee.
+    char * operator()(const char * literal /*c-string*/){
+        str.reset();
+        for(;*literal != 0; literal++){
+            append(*literal);
+        }
+        close_quoting();    // P4 exception, to close whole quoting
+        return str;
+    }
+};
+    
+class WeightedRand{
+// Return a random number in [0, size)
+// Every number has different chance (aka weight) to be selected.
+private:
+    Buffer_int weights;
+    double total;
+    WeightedRand(const WeightedRand &);
+    WeightedRand & operator = (const WeightedRand &);
+public:
+    WeightedRand(Buffer_int * weight_list = NULL, int size = 0){
+        if ( weight_list == NULL){
+            for (int i=0; i<size; ++i) weights.append(DEFAULT_WEIGHT);
+        } else {
+            int s = weight_list->content_size();
+            if (s < size){
+                weights.append_array( (*weight_list),s);
+                for (int i=s; i<size; ++i) weights.append(DEFAULT_WEIGHT);
+            } else { // s >= size
+                weights.append_array( (*weight_list),size);
+            }
+        }
+        total = 0;
+        int c = weights.content_size();
+        for (int i=0; i<c; ++i){
+            total += weights[i];
+        }
+    }
+
+    void append(int weight){
+        weights.append(weight);
+        total += weight;
+    }
+
+    // Give a random number with the consideration of weight.
+    // Every random number is associated with a weight.
+    // It identifies the chance to be selected, 
+    // larger weight has more chance to be selected.
+    //
+    //
+    //  ______________________   every slot has equal chance
+    //
+    //  [____][_][___][______]   each item has different slots, hence different chance
+    //
+    //
+    //  The algorithms to generate the number is illustrated by preceding figure.
+    //  First, a slot is selected by rand(). Then we translate the slot to corresponding item.
+    //
+    int next(){
+        // get a random in [0,1]
+        double reference_mark = (double)rand() / (double)RAND_MAX;
+
+        // get the slot's index, 0 <= mark <= total;
+        double mark = total * reference_mark;
+
+        // translate the slot to corresponding item
+        int i=0;
+        for (;;){
+            mark -= weights[i];  // 0 <= mark <= total
+            if (mark <= 0)
+                break;
+            i++;
+        }
+        return i;
+    }
+};
+    
+///////////////////////////////////////////////////////////
+//
+// The parser result nodes
+//
+    
+class Literal : public Pick {
+public:
+    virtual const char* next(){
+        return str;
+    }
+    Literal(const char * s /*c-string*/){
+        str.append_array(s, strlen(s) + 1);
+    }
+private:
+    Buffer_char str; //null-terminated c-string
+};
+
+class Variable : public Pick {
+public:
+    Variable(SymbolTable * symbols, const char * varName, Pick * varRef = NULL){
+        this->var_name.append_array(varName, strlen(varName) + 1);
+        if ((symbol_table = symbols)){
+            symbol_table->put(varName, varRef);
+        }
+    }
+
+    operator const char *(){
+        return var_name;
+    }
+
+    virtual const char* next(){
+        if (symbol_table){
+            Pick * var_ref = NULL;
+            symbol_table->find(var_name, &var_ref);
+            if (var_ref) {
+                return var_ref->next();
+            }
+        }
+        return "";  // dumb string
+    }
+private:
+    Buffer_char var_name;
+    SymbolTable * symbol_table;
+};
+
+class Quote : public Pick{
+public:
+    Quote(Pick & base):item(base),e(Escaper::NO, Escaper::NO, Escaper::BSLASH_ONLY){
+    }
+    virtual const char* next(){
+        return e(item.next());
+    }
+private:
+    Pick & item;
+    Buffer_char str;
+    Escaper e;
+};
+
+
+class Morph : public Pick{
+/*
+The difference between morph and an arbitrary random string is that 
+a morph changes slowly. When we build collation rules, for example, 
+it is a much better test if the strings we use are all in the same 
+'neighborhood'; they share many common characters.
+*/
+public:
+    Morph(Pick & base):item(base){}
+
+    virtual const char* next(){
+        current.reset();
+        const char * s = item.next();
+        current.append_array(s, strlen(s) + 1);
+        if  (last.content_size() == 0) {
+            str.reset();
+            last.reset();
+            str.append_array(current, current.content_size());
+            last.append_array(current, current.content_size());
+        } else {
+            morph();
+        }
+        return str;
+    }
+private:
+    Pick & item;
+    Buffer_char str;
+    Buffer_char last;
+    Buffer_char current;
+
+    char * p_last;
+    char * p_curr;
+
+    void copy_curr(){
+        if (*p_curr) {
+            str.append(*p_curr);
+            p_curr++;
+        }
+    }
+
+    void copy_last(){
+        if (*p_last) {
+            str.append(*p_last);
+            p_last++;
+        }
+    }
+
+    // copy 0, 1, or 2 character(s) to str
+    void copy(){
+        static WeightedRand wr(& Buffer_int().append(DEFAULT_WEIGHT * 10), 5);
+
+        switch (wr.next()){
+            case 0: // copy last  -- has 10 times chance than others
+                copy_last();
+                break;
+            case 1: // copy both
+                copy_curr();
+                copy_last();
+                break;
+            case 2: // copy both
+                copy_last();
+                copy_curr();
+                break;
+            case 3:
+                copy_curr();
+                break;
+            case 4:  // copy nothing
+                break;
+            default:
+                // ASSERT(FALSE);
+                ;
+        }
+    }
+
+    void morph(void){
+        int min = strlen(last);
+        int max = strlen(current);
+        if (min > max){
+            int temp  = min;
+            min = max;
+            max = temp;
+        }
+
+        int len = min + rand()%(max - min + 1); // min + [0, diff]
+        p_curr = current;
+        p_last = last;
+        str.reset();
+
+        for (; str.content_size()<len && *p_curr && *p_last;){
+            copy(); // copy 0, 1, or 2 character(s) to str
+        }
+
+        if (str.content_size() == len) {
+            str.append(0);
+            final();
+            return;
+        }
+
+        if (str.content_size() > len) { // if the last copy copied two characters
+            str[len]=0;
+            final();
+            return;
+        }
+
+        // str.content_size() < len
+        if (*p_last) {
+            for (; str.content_size() < len; copy_last());
+        } else if (*p_curr){
+            for (; str.content_size() < len; copy_curr());
+        }
+
+        int last_len = last.content_size();
+        for (;str.content_size() < len;){
+            str.append(last[rand()%last_len]);
+        }
+        str.append(0);
+        final();
+    }
+
+    void final(){
+        last.reset();
+        last.append_array(current, current.content_size());
+    }
+};
+
+class Sequence : public Pick {
+public:
+    virtual const char* next(){
+        str.reset();
+        int s = items.content_size();
+        for(int i=0; i < s; i++){
+            const char * t = items[i]->next();
+            str.append_array(t, strlen(t));
+        }
+        str.append(0); // terminal null
+        return str;
+    }
+
+    void append (Pick * node){
+        items.append(node);
+    }
+
+    virtual ~Sequence(){
+        int s = items.content_size();
+        for(int i=0; i < s; i++){
+            //How can assure the item is got from heap?
+            //Let's assume it.
+            delete items[i]; // TOFIX: point alias/recursion problem
+            items[i] = NULL;
+        }
+    }
+private:
+    Buffer_pPick items;
+    Buffer_char  str; //null-terminated c-string
+};
+
+class Repeat : public Pick {
+private:
+    Pick * item;
+    Buffer_char str;
+    WeightedRand wr;
+    int min;
+    int max;
+    int select_a_count(){
+        return min + wr.next();
+    }
+public:
+    virtual const char* next(){
+        str.reset();
+        int c = select_a_count();
+        for(int i=0; i< c; i++){
+            const char * t = item->next();
+            str.append_array(t, strlen(t));
+        }
+        str.append(0);
+        return str;
+    }
+
+    Repeat(Pick * base, int minCount =0, int maxCount = 1, Buffer_int * weights = NULL):
+        wr(weights, maxCount-minCount +1) {
+        this->item = base;
+        this->min = minCount;
+        this->max = maxCount;
+    }
+    virtual ~Repeat(){
+        delete item;  // TOFIX: point alias/recursion problem
+        item = NULL;
+    }
+};
+
+
+class Alternation : public Pick {
+public:
+    virtual const char* next(){
+        str.reset();
+        int i = wr.next();
+        const char * t = items[i]->next();
+        str.append_array(t, strlen(t) + 1);
+        return str;
+    }
+    virtual ~Alternation(){
+        int s = items.content_size();
+        for(int i=0; i < s; i++){
+            delete items[i];  // TOFIX: point alias/recursion problem
+            items[i] = NULL;
+        }
+    }
+
+    Alternation & append (Pick * node, int weight = DEFAULT_WEIGHT){
+        items.append(node);
+        wr.append(weight);
+        return *this;
+    }
+private:
+    Buffer_pPick items;
+    Buffer_char str; // null-terminated c-string
+    WeightedRand wr;
+};
+    
+///////////////////////////////////////////////////////////
+//
+// The parser
+//
+
+enum TokenType {STRING, VAR, NUMBER, STREAM_END, ERROR, QUESTION, STAR, PLUS, LBRACE, RBRACE, LPAR, RPAR, SEMI, EQ, COMMA, BAR, AT, WAVE, PERCENT};
+
+class Scanner{
+friend int DumpScanner(Scanner & s, UBool dumb);
+private:
+    const char * source;
+    const char * working;
+    const char * history; // for debug
+    enum StateType {START, IN_NUM, IN_VAR_FIRST, IN_VAR, IN_QUOTE, IN_QUOTE_BSLASH, IN_BSLASH, IN_STRING, DONE};
+    StateType state;
+    void terminated(TokenType t){
+        working--;       // return the peeked character
+        tokenType = t;
+        token.append(0); // close buffer
+        state = DONE;
+    }
+public:
+    // the buffer of "source" is owned by caller
+    Scanner(const char *src/*[in] c-string*/ = NULL):source(src){
+        working = src;
+        history = working;
+        state = DONE;
+        tokenType = ERROR;
+    }
+
+    //void setSource(const char *const src /*[in] c-string*/){
+    //    *(&const_cast<const char *>(source)) = src;
+    //}
+
+    Buffer_char token;
+    TokenType tokenType;
+
+    TokenType getNextToken(){
+        token.reset();
+        state = START;
+        history = working; // for debug
+        while (state != DONE){
+            char c = *working++;
+            if (c == 0 && state != START){//avoid buffer overflow. for IN_QUOE, IN_ESCAPE
+                terminated(ERROR);
+                break; // while
+            }
+            switch(state){
+                case START:
+                    tokenType = ERROR;
+                    switch(c){
+                        case '?'  : tokenType = QUESTION; break;
+                        case '*'  : tokenType = STAR; break;
+                        case '+'  : tokenType = PLUS; break;
+                        case '{'  : tokenType = LBRACE; break;
+                        case '}'  : tokenType = RBRACE; break;
+                        case '('  : tokenType = LPAR; break;
+                        case ')'  : tokenType = RPAR; break;
+                        case ';'  : tokenType = SEMI; break;
+                        case '='  : tokenType = EQ; break;
+                        case ','  : tokenType = COMMA; break;
+                        case '|'  : tokenType = BAR; break;
+                        case '@'  : tokenType = AT; break;
+                        case '~'  : tokenType = WAVE; break;
+                        case '%'  : tokenType = PERCENT; break;
+                        case 0    : tokenType = STREAM_END; working-- /*avoid buffer overflow*/; break;
+                    }
+                    if (tokenType != ERROR){
+                        token.append(c);
+                        token.append(0);
+                        state = DONE;
+                        break; // START
+                    }
+                    switch(c){
+                        case '$'  : state = IN_VAR_FIRST; token.append(c); break;
+                        case '\'' : state = IN_QUOTE;     break;
+                        case '\\' : state = IN_BSLASH;    break;
+                        default:
+                            if (isWhiteSpace(c)){    // state = START;   //do nothing
+                            } else if (isDigit(c)){     state = IN_NUM;    token.append(c);
+                            } else if (isAlphabet(c)){  state = IN_STRING; token.append(c);
+                            } else {terminated(ERROR);}
+                    }
+                    break;//START
+                case IN_NUM:
+                    if (isDigit(c)){
+                        token.append(c);
+                    } else {
+                        terminated(NUMBER);
+                    }
+                    break;//IN_NUM
+                case IN_VAR_FIRST:
+                    if (isAlphabet(c)){
+                        token.append(c);
+                        state = IN_VAR;
+                    } else {
+                        terminated(ERROR);
+                    }
+                    break; // IN_VAR_FISRT
+                case IN_VAR:
+                    if (isAlphabet(c) || isDigit(c)){
+                        token.append(c);
+                    } else {
+                        terminated(VAR);
+                    }
+                    break;//IN_VAR
+                case IN_STRING:
+                    // About the scanner's behavior for STRING, AT, and ESCAPE:
+                    // All of them can be contacted with each other. 
+                    // This means the scanner will eat up as much as possible strings
+                    //   (STRING, AT, and ESCAPE) at one time, with no regard of their
+                    //   combining sequence.
+                    //
+                    if (c == '\''){
+                        state = IN_QUOTE; // the first time we see single quote
+                    } else if (c =='\\'){ // back slash character
+                        state = IN_BSLASH;
+                    } else if (isAlphabet(c) || isDigit(c)){
+                        token.append(c);
+                    } else{
+                        terminated(STRING);
+                    }
+                    break;//IN_STRING
+                case IN_QUOTE:
+                    if (c == '\''){ // the second time we see single quote
+                        state = IN_STRING; // see document in IN_STRING
+                    } else if ( c== '\\') { // backslah escape in quote
+                        state = IN_QUOTE_BSLASH;
+                    } else {
+                        token.append(c);  // eat up everything, includes back slash
+                    }
+                    break;//IN_QUOTE
+                case IN_QUOTE_BSLASH:
+                case IN_BSLASH:
+                    switch (c){
+                        case 'n'  : token.append('\n'); break;
+                        case 'r'  : token.append('\r'); break;
+                        case 't'  : token.append('\t'); break;
+                        case '\'' : token.append('\''); break;
+                        case '\\' : token.append('\\'); break;
+                        default: token.append(c); // unknown escaping, treat it as literal
+                    }
+                    if (state == IN_BSLASH){
+                        state = IN_STRING; // see document in IN_STRING
+                    } else { // state == IN_QUOTE_BSLASH
+                        state = IN_QUOTE;
+                    }
+                    break;//IN_BSLASH
+                case DONE:  /* should never happen */
+                default:
+                    working--;
+                    tokenType = ERROR;
+                    state = DONE;
+                    break;
+            }//switch(state) 
+        }//while (state != DONE)
+
+        return tokenType;
+    }
+};//class Scanner
+
+class Parser{
+friend UBool TestParser();
+friend class TestParserT;
+friend class LanguageGenerator_impl;
+private:
+    Scanner s;
+    TokenType & token;
+    int min_max;   // for the evil infinite
+    
+    UBool match(TokenType expected){
+        if (token == expected) {
+            token = s.getNextToken();
+            return TRUE;
+        } else {
+            //s.dumpCurrentPoint();
+            return FALSE;
+        }
+    }
+
+    UBool weight(int & value){
+        if (token == NUMBER){
+            int temp = atoi(s.token);
+            match(NUMBER);
+            if (match(PERCENT)){
+                value = temp;
+                return TRUE;
+            }
+        }
+        return FALSE;
+    }
+
+    UBool repeat (Pick* &node /*in,out*/){
+        if (node == NULL) return FALSE;
+
+        int count = -2;
+        int min = -2;
+        int max = -2;
+        UBool question = FALSE;
+        switch (token){
+            case QUESTION:
+                match(QUESTION);
+                min = 0;
+                max = 1;
+                count = 2;
+                question = TRUE;
+                break;
+            case STAR:
+                match(STAR);
+                min = 0;
+                max = -1;
+                count = -1;
+                break;
+            case PLUS:
+                match(PLUS);
+                min = 1;
+                max = -1;
+                count = -1;
+                break;
+            case LBRACE:
+                match(LBRACE);
+                if (token != NUMBER){
+                    return FALSE;
+                }else {
+                    min = atoi(s.token);
+                    match(NUMBER);
+                    if (token == RBRACE){
+                        match(RBRACE);
+                        max = min;
+                        count = 1;
+                    } else if (token == COMMA) {
+                        match(COMMA);
+                        if (token == RBRACE){
+                            match(RBRACE);
+                            max = -1;
+                            count = -1;
+                        } else if (token == NUMBER) {
+                            max = atoi(s.token);
+                            match(NUMBER);
+                            count = max - min + 1;
+                            if (!match(RBRACE)) {
+                                return FALSE;
+                            }
+                        } else {
+                            return FALSE;
+                        }
+                    } else {
+                        return FALSE;
+                    }
+                }
+                break;
+            default:
+                return FALSE;
+        }
+
+        if (count == -2 || min == -2 || max == -2){
+            //ASSERT(FALSE);
+            return FALSE;
+        }
+
+        // eat up following weights
+        Buffer_int weights;
+        int w;
+        while (weight(w)){
+            weights.append(w);
+        }
+
+        // for the evil infinite
+        min_max = min_max > min ? min_max : min;
+        min_max = min_max > max ? min_max : max;
+        if (min_max > PSEUDO_INFINIT){
+            return FALSE; // PSEUDO_INFINIT is less than the real maximum
+        }
+        if (max == -1){ // the evil infinite
+            max = PSEUDO_INFINIT;
+        }
+        // for the strange question mark
+        if (question && weights.content_size() > 0){
+            Buffer_int w2;
+            w2.append(DEFAULT_WEIGHT - weights[0]).append(weights[0]);
+            node = new Repeat(node,min,max,&w2);
+            return TRUE;
+        }
+        node = new Repeat(node,min,max,&weights);
+        return TRUE;
+    }
+
+    UBool core(Pick* &node /*out*/){
+        if (node != NULL) return FALSE; //assert node == NULL
+
+        switch(token){
+            case LPAR:
+                match(LPAR);
+                if(defination(node) && match(RPAR)){
+                    return TRUE;
+                }
+                return FALSE;
+            case VAR:
+                node = new Variable(&symbols, s.token);
+                match(VAR);
+                return TRUE;
+            case STRING:
+                node = new Literal(s.token);
+                match(STRING);
+                return TRUE;
+            default:
+                return FALSE;
+        }
+    }
+    UBool modified(Pick* &node /*out*/){
+        if (node != NULL) return FALSE; //assert node == NULL
+
+        if (!core(node)) {
+            return FALSE;
+        }
+
+        for (;;){
+            switch(token){
+                case WAVE:
+                    match(WAVE);
+                    node = new Morph(*node);
+                    break;
+                case AT:
+                    match(AT);
+                    node = new Quote(*node);
+                    break;
+                case QUESTION:
+                case STAR:
+                case PLUS:
+                case LBRACE:
+                    if (!repeat(node)) return FALSE;
+                    break;
+                case SEMI:      // rule definiation closed
+                case RPAR:      // within parenthesis (core closed)
+                case BAR:       // in alternation
+                case NUMBER:    // in alternation, with weight
+                case LPAR:      // in sequence
+                case VAR:       // in sequence
+                case STRING:    // in sequence
+                    return TRUE;
+                default:
+                    return FALSE;
+            }
+        }
+    }
+
+
+    UBool sequence_list(Pick* &node /*in,out*/){
+        if (node == NULL) return FALSE; // assert node != NULL
+
+        Sequence* seq = new Sequence();
+        Pick * n = node;
+
+        while (token == VAR || token == STRING || token == LPAR){
+            seq->append(n);
+            n = NULL;
+            if (modified(n)){
+                // go on
+            } else {
+                goto FAIL;
+            }
+        }
+
+        if (token == SEMI || token == RPAR || token == BAR){
+            seq->append(n);
+            node = seq;
+            return TRUE;
+        }
+FAIL:
+        delete seq;
+        return FALSE;
+
+    }
+
+    UBool sequence(Pick* &node /*out*/){
+        if (node != NULL) return FALSE; //assert node == NULL
+
+        if (!modified(node)) {
+            return FALSE;
+        }
+
+        if (token == VAR || token == STRING || token == LPAR){
+            return sequence_list(node);
+        } else {
+            return TRUE; // just a modified
+        }
+    }
+
+    UBool alternation_list(Pick* &node /*in,out*/){
+        if (node == NULL) return FALSE; // assert node != NULL
+
+        Alternation * alt = new Alternation();
+        Pick * n = node;
+        int w = DEFAULT_WEIGHT;
+
+        while (token == NUMBER || token == BAR){
+            if(token == NUMBER) {
+                if (weight(w)){ 
+                    if (token == BAR){ 
+                        // the middle item, go on
+                    } else {
+                        // the last item or encounter error
+                        break; //while
+                    } 
+                } else {
+                    goto FAIL;
+                }
+            } // else token == BAR
+            match(BAR);
+            alt->append(n,w);
+
+            n = NULL;
+            w = DEFAULT_WEIGHT;
+            if (sequence(n)){
+                // go on
+            } else {
+                goto FAIL;
+            }
+        }
+
+        if (token == SEMI || token == RPAR) {
+            alt->append(n,w);
+            node = alt;
+            return TRUE;
+        }
+FAIL:
+        delete alt;
+        return FALSE;
+    }
+
+    UBool alternation(Pick* &node /*out*/){
+        if (node != NULL) return FALSE; //assert node == NULL
+
+        // 'sequence' has higher precedence than 'alternation'
+        if (!sequence(node)){
+            return FALSE;
+        }
+
+        if (token == BAR || token == NUMBER){ // find a real alternation1, create it.
+            return alternation_list(node);
+        } else {
+            return TRUE;    // just a sequence_old
+        }
+    }
+
+
+    UBool defination(Pick* &node /*out*/){
+        if (node != NULL) return FALSE; //assert node == NULL
+        return alternation(node);
+    }
+
+    UBool rule(){
+        if (token == VAR){
+            Buffer_char name;
+            name.append_array(s.token, strlen(s.token) + 1);
+            match(VAR);
+
+            if (match(EQ)){
+                Pick * t = NULL;
+                if(defination(t)){
+                    symbols.put(name, t);
+                    return match(SEMI);
+                }
+            }
+        }
+        return FALSE;
+    }
+public:
+    UBool rules(){
+        symbols.reset();
+        token = s.getNextToken();
+        while (rule()){
+        }
+        if (token == STREAM_END){
+            return TRUE;
+        } else {
+            //s.dumpCurrentPoint();
+            return FALSE;
+        }
+    }
+
+public:
+    SymbolTable symbols;
+
+    Parser(const char *const source):s(source), token(s.tokenType){
+        min_max = -2;
+    }
+    UBool parse(){
+        return rules();
+    }
+
+}; // class Parser
+        
+
+///////////////////////////////////////////////////////////
+//
+// 
+//
+
+int DumpScanner(Scanner & s, UBool dump = TRUE){
+    int len = strlen(s.source);
+    int error_start_offset = s.history - s.source;
+    if (dump){
+        printf("\n=================== DumpScanner ================\n");
+        fwrite(s.source, len, 1, stdout);
+        printf("\n-----parsed-------------------------------------\n");
+        fwrite(s.source, s.history - s.source, 1, stdout);
+        printf("\n-----current------------------------------------\n");
+        fwrite(s.history, s.working - s.history, 1, stdout);
+        printf("\n-----unparsed-----------------------------------\n");
+        fwrite(s.working, (s.source + len - s.working), 1, stdout);
+        printf("\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n");
+    }
+    return error_start_offset;
+}
+
+class LanguageGenerator_impl{
+public:
+    LanguageGenerator_impl(const char *const bnf_definition, const char *const top_node)
+        :par(bnf_definition), top_node_name(top_node){
+        srand((unsigned)time( NULL ));
+    }
+
+    LanguageGenerator::PARSE_RESULT parseBNF(UBool debug = TRUE){
+        if (par.parse()){
+            if (par.symbols.find(top_node_name, &top_node_ref) == SymbolTable::HAS_REF) {
+                if (par.symbols.is_complete()) {
+                    return LanguageGenerator::OK;
+                } else {
+                    if (debug) printf("The bnf definition is incomplete.\n");
+                    return LanguageGenerator::INCOMPLETE;
+                }
+            } else {
+                if (debug) printf("No top node is found.\n");
+                return LanguageGenerator::NO_TOP_NODE;
+            }
+        } else {
+            if(debug) {
+                printf("The bnf definition is wrong\n");
+                DumpScanner(par.s, TRUE);
+            }
+            return LanguageGenerator::BNF_DEF_WRONG;
+        }
+    }
+    const char * next(){
+        return top_node_ref->next();
+    }
+
+private:
+    Parser par;
+    const char *const top_node_name;
+    Pick * top_node_ref;
+};
+
+LanguageGenerator::LanguageGenerator():lang_gen(NULL){
+}
+
+LanguageGenerator::~LanguageGenerator(){
+    delete lang_gen;
+}
+
+LanguageGenerator::PARSE_RESULT LanguageGenerator::parseBNF(const char *const bnf_definition /*in*/, const char *const top_node/*in*/, UBool debug){
+    if (lang_gen){
+        delete lang_gen;
+    }
+    lang_gen = new LanguageGenerator_impl(bnf_definition, top_node);
+    PARSE_RESULT r = lang_gen->parseBNF(debug);
+    if (r != OK){
+        delete lang_gen;
+        lang_gen = NULL;
+        return r;
+    } else {
+        return r;
+    }
+}
+const char *LanguageGenerator::next(){ // Return a null-terminated c-string. The buffer is owned by callee.
+    if (lang_gen){
+        return lang_gen->next();
+    }else { 
+        return "";
+    }
+}
+
+///////////////////////////////////////////////////////////
+//
+// The test code for WBNF
+//
+
+#define CALL(fun) \
+    if (fun()){ \
+        printf("Pass: " #fun "\n");\
+    } else { \
+        printf("FAILED: !!! " #fun " !!!\n"); \
+    }
+
+#define DUMP_R(fun, var, times) \
+    {printf("\n========= " #fun " =============\n"); \
+    for (int i=0; i<times; i++) { \
+        const char * t = var.next();\
+        fwrite(t,strlen(t),1,stdout); \
+        printf("\n");   \
+    }   \
+    printf("^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n");}
+
+
+
+#if TEST_WBNF_TEST    
+static UBool TestQuote(){
+    const char *const str = "This ' A !,z| qq [] .new\tline";
+    //const char *const str_r = "This \\' A '!,'z'|' qq '[]' '.'new\tline";
+    ////
+    //// :(  we must quote our string to following C syntax
+    ////     cannot type the literal here, it makes our code rather human unreadable
+    ////     very very unconformable!
+    ////
+    ///* 
+    //*/
+
+    //const char *const s1    =   "ab'c";
+    //const char (* s1_r1) [] = { "ab''c",    // ab''c
+    //                            "ab\\'c",   // ab\'c
+    //                           };//
+    ///*
+    // .      '.'     \.
+    // ..     \.\.    '.'\.   '.'\.   '..'    // '.''.'  wrong
+    //*/
+
+    //const char *const s2    =   "a..'.b";       // a..'.b
+    //const char (*s2_r) []   = { "a'..''.'b"     // a'..''.'b   
+    //                           ,"a'..\\'.'b"    // a'..\'.'b
+    //                           ,"a'..'\\''.'b"  // a'..'\''.'b
+    //                          };//
+
+    //const char *const s3    =   "a..\\.b";      // a..\.b
+    //const char (*s3_r) []   = { "a'..\\\\.'b"   // a'..\\.'b
+    //                           ,"a'..'\\\\'.'b" // a'..'\\'.'b
+    //                          };//
+
+    //                            // no catact operation, no choice, must be compact
+
+    srand((unsigned)time( NULL ));
+    
+    //Escaper l(Escaper::NO, Escaper::NO, Escaper::RAND_ESC);
+    Pick *p = new Literal(str);
+    Quote q(*p);
+
+    DUMP_R(TestQuote, (*p), 1);
+    DUMP_R(TestQuote, q, 20);
+    return FALSE;
+}
+static UBool TestLiteral(){
+    const char * s = "test string99.";
+    Literal n(s);
+    const char * r = n.next();
+    return strcmp(s,r) == 0;
+}
+
+static UBool TestSequence(){
+    Sequence seq;
+    seq.append(new Literal("abc "));
+    seq.append(new Literal(", s"));
+
+    return strcmp(seq.next(), "abc , s") == 0;
+}
+static UBool TestAlternation(){
+    srand((unsigned)time( NULL ));
+    Alternation alt;
+    alt.append(new Literal("aaa_10%"),10);
+    alt.append(new Literal("bbb_0%"),0);
+    alt.append(new Literal("ccc_10%"),10);
+    alt.append(new Literal("ddddddd_50%"),50);
+
+    DUMP_R(TestAlternation, alt, 50);
+
+    return FALSE;
+}
+
+static UBool TestBuffer(){
+    Buffer_int t;
+    t.append(1).append(0).append(5);
+    int s = t.content_size();
+    for (int i=0; i<s; ++i){
+        printf("%d\n", t[i]);
+    }
+    return FALSE;
+}
+
+static UBool TestWeightedRand(){
+    srand((unsigned)time( NULL ));
+    Buffer_int t;
+    t.append(1).append(0).append(5);
+    WeightedRand wr(&Buffer_int().append(10).append(0).append(50),4);
+//    WeightedRand wr(&t,3);
+    for (int i=0; i< 50; ++i){
+        printf("%d\n", wr.next());
+    }
+    return FALSE;
+}
+
+static UBool TestRepeat(){
+    srand((unsigned)time( NULL ));
+    Repeat rep(new Literal("aaa1-5 "), 1, 5);
+    DUMP_R(TestRepeat, rep, 50);
+
+    Repeat r2(new Literal("b{1,3}1%0%5% "), 1, 3, &Buffer_int().append(1).append(0).append(5));
+    DUMP_R(TestRepeat, r2, 50);
+
+    Repeat r3(new Literal("aaa5-5 "), 5, 5);
+    DUMP_R(TestRepeat, r3, 50);
+
+    return FALSE;
+}
+
+static UBool TestVariable(){
+    SymbolTable tab;
+    Pick * value = new Literal("string1");
+    Variable var1(&tab, "x", value);
+
+    Variable var2(&tab, "y");
+//    tab.put(var2, value); // TOFIX: point alias/recursion problem
+    Pick * value2 = new Literal("string2");
+    tab.put(var2, value2);
+
+    Pick * value3 = new Literal("string3");
+    Variable var3(&tab, "z");
+    tab.put("z", value3);
+
+    UBool pass;
+    pass = strcmp(var1.next(), value->next()) == 0;
+    pass = pass && strcmp(var2.next(), value2->next()) == 0;
+    pass = pass && strcmp(var3.next(), value3->next()) == 0;
+    return pass;
+}
+
+static UBool TestSymbolTable(){
+    Literal * n1 = new Literal("string1");
+    Literal * n2 = new Literal("string2");
+    SymbolTable t;
+    t.put("abc", n1);
+    t.put("$aaa", n2);
+//    t.put("alias", n1);  // TOFIX: point alias/recursion problem
+    t.put("bbb");
+
+    UBool pass;
+    pass = t.find(NULL) == SymbolTable::EMPTY;
+    pass = pass && t.find("ccc") == SymbolTable::NO_VAR;
+    pass = pass && t.find("bbb") == SymbolTable::NO_REF;
+    pass = pass && t.find("abc") == SymbolTable::HAS_REF;
+    pass = pass && t.find("$aaa") == SymbolTable::HAS_REF;
+
+    t.reset();
+    pass = pass && t.find("abc") == SymbolTable::NO_VAR;
+    return pass;
+}
+
+
+static UBool TestScanner(void){
+    //const char str1[] = "$root = $command{0,5} $reset $mostRules{1,20};";
+    //const char str1_r[][20] = {"$root", "=", "$command", "{", "0", ",", "5", "}", 
+    //    "$reset", "$mostRules", "{", "1", ",", "20", "}", ";"};
+
+    const char str2[] = "$p2 =(\\\\ $s $string $s)? 25%;";
+    const char str2_r[][20] = {"$p2", "=", "(", "\\", "$s", "$string", "$s", ")", "?", "25", "%", ";"};
+
+    const char *str = str2;
+    const char (*str_r)[20] = str2_r;
+    int tokenNum = sizeof(str2_r)/sizeof(char[20]);
+
+    Scanner t(str);
+    UBool pass = TRUE;
+    t.getNextToken();
+    int i = 0;
+    while (pass){
+        if (t.tokenType == STREAM_END){
+            pass = pass? i == tokenNum : FALSE;
+            break;//while
+        } else if (t.tokenType == ERROR){
+            pass = FALSE;
+            break;//while
+        } else {
+            pass = strcmp( &(t.token[0]), str_r[i++]) == 0;
+            t.getNextToken();
+        }
+    }
+
+    //const char ts[] = "$commandList = '['"
+    //" ( alternate ' ' $alternateOptions"
+    //" | backwards ' 2'"
+    //" | normalization ' ' $onoff "
+    //" | caseLevel ' ' $onoff "
+    //" | hiraganaQ ' ' $onoff"
+    //" | caseFirst ' ' $caseFirstOptions"
+    //" | strength ' ' $strengthOptions"
+    //" ) ']';" ;
+
+    //Scanner t2(ts);
+    //pass = TRUE;
+    //do {
+    //    t2.getNextToken();
+    //    if (t2.tokenType == ERROR){
+    //        DumpScanner(t2);
+    //        return FALSE;
+    //    }
+    //}while (t.tokenType != STREAM_END);
+
+    return pass;
+}
+
+class TestParserT {
+public:
+UBool operator () (const char *const str, const int exp_error_offset = -1, const UBool dump = TRUE){
+    Parser par(str);
+    if (par.rules()){
+        if ( exp_error_offset == -1){
+            return TRUE;
+        }else {
+            DumpScanner(par.s,dump);
+            return FALSE;
+        }
+    }else {
+        return DumpScanner(par.s, dump) == exp_error_offset;
+    }
+}
+};
+
+UBool TestParser(){
+    TestParserT test;
+
+    UBool pass = TRUE;
+    pass = pass && test ("$s = ' ' ? 50%;");
+    pass = pass && test("$x = ($var {1,2}) 3%;");         // legal
+    pass = pass && test("$x = $var {1,2} 3% | b 4%;");    // legal
+    pass = pass && test("$x = $var {1,2} 3%;");           // legal
+    pass = pass && test("$m = $c ? 2% 4% | $r 5% | $n 25%;"); // legal
+    pass = pass && test("$a = b ? 2% | c 5%;");               // legal
+    pass = pass && test("$x = A B 5% C 10% | D;", 8, FALSE);  // illegal 5%
+    pass = pass && test("$x = aa 45% | bb 5% cc;", 19, FALSE);// illegal cc
+    pass = pass && test("$x = (b 5%) (c 6%);");               // legal
+    pass = pass && test("$x = (b 5%) c 6%;", 13, FALSE);      // illegal 6%
+    pass = pass && test("$x = b 5% (c 6%);", 9, FALSE);       // illegal (c 6%)
+    pass = pass && test("$x = b 5% c 6%;", 9, FALSE);         // illegal c 6%
+    pass = pass && test("$x = b 5%;");                        // legal
+    pass = pass && test("$x = aa 45% | bb 5% cc;", 19, FALSE);// illegal cc
+    pass = pass && test("$x = a | b  | c 4% | d 5%;");        // legal
+    pass = pass && test("$s = ' ' ? 50% abc;");               // legal
+    pass = pass && test("$s =  a | c d | e f;");              // legal
+    pass = pass && test( "$z = q 0% | p 1% | r 100%;");         // legal How to check parsed tree??
+
+    pass = pass && test("$s = ' ' ? 50%;");
+    pass = pass && test("$relationList = '<' | '<<' |  ';' | '<<<' | ',' | '=';");
+    pass = pass && test("$p1 = ($string $s '|' $s)? 25%;");
+    pass = pass && test("$p2 = (\\\\ $s $string $s)? 25%;");
+    pass = pass && test("$rel2 = $p1 $string $s $p2;");
+    pass = pass && test("$relation = $relationList $s ($rel1 | $rel2) $crlf;");
+    pass = pass && test("$command = $commandList $crlf;");
+    pass = pass && test("$reset = '&' $s ($beforeList $s)? 10% ($positionList 100% | $string 10%) $crlf;");
+    pass = pass && test("$mostRules = $command 1% | $reset 5% | $relation 25%;");
+    pass = pass && test("$root = $command{0,5} $reset $mostRules{1,20};");
+
+    const char collationBNF[] =
+    "$s = ' '? 50%;" 
+    "$crlf = '\r\n';" 
+
+    "$alternateOptions = non'-'ignorable | shifted;" 
+    "$onoff = on | off;" 
+    "$caseFirstOptions = off | upper | lower;" 
+    "$strengthOptions = '1' | '2' | '3' | '4' | 'I';" 
+    "$commandList = '['"
+    " ( alternate ' ' $alternateOptions"
+    " | backwards ' 2'"
+    " | normalization ' ' $onoff "
+    " | caseLevel ' ' $onoff "
+    " | hiraganaQ ' ' $onoff"
+    " | caseFirst ' ' $caseFirstOptions"
+    " | strength ' ' $strengthOptions"
+    " ) ']';" 
+    "$command = $commandList $crlf;" 
+
+    "$ignorableTypes = (tertiary | secondary | primary) ' ' ignorable;" 
+    "$allTypes = variable | regular | implicit | trailing | $ignorableTypes;" 
+    "$positionList = '[' (first | last) ' ' $allTypes ']';"
+
+    "$beforeList = '[before ' ('1' | '2' | '3') ']';"
+
+    "$relationList = ("
+    "   '<'"
+    " | '<<'"
+    " | ';'" 
+    " | '<<<'"
+    " | ','" 
+    " | '='"
+    ");"
+    "$string = $magic;" 
+    "$rel1 = '[variable top]' $s;" 
+    "$p1 = ($string $s '|' $s)? 25%;" 
+    "$p2 = (\\\\ $s $string $s)? 25%;" 
+    "$rel2 = $p1 $string $s $p2;" 
+    "$relation = $relationList $s ($rel1 | $rel2) $crlf;" 
+
+    "$reset = '&' $s ($beforeList $s)? 10% ($positionList 1% | $string 10%) $crlf;" 
+    "$mostRules = $command 1% | $reset 5% | $relation 25%;"
+    "$root = $command{0,5} $reset $mostRules{1,20};"
+    ;
+    
+    pass = pass && test(collationBNF);
+
+
+    return pass;
+}
+
+static UBool TestMorph(){
+    srand((unsigned)time( NULL ));
+
+    Alternation * alt = new Alternation();
+
+    (*alt)
+    .append(new Literal("a")).append(new Literal("b")).append(new Literal("c"))
+    .append(new Literal("d")).append(new Literal("e")).append(new Literal("f"))
+    .append(new Literal("g")).append(new Literal("h")).append(new Literal("i"))
+    .append(new Literal("j")).append(new Literal("k")).append(new Literal("l"))
+    .append(new Literal("m")).append(new Literal("n")).append(new Literal("o"))
+    ;
+
+    Repeat * rep = new Repeat( alt ,5,5 );
+    Morph m( *rep);
+
+//    DUMP_R(TestMorph,(*rep),20);
+    DUMP_R(TestMorph,m,100);
+
+    return FALSE;
+}
+
+#endif
+
+static UBool TestLanguageGenerator(){
+    //LanguageGenerator g;
+    //const char *const s = "$s = p 0% | q 1%;";
+    //g.parseBNF(s, "$s");
+    UBool pass;
+    //= strcmp("q", g.next()) == 0;
+
+    const char *const def = 
+        //"$a = $b;"
+        //"$b = $c;"
+        //"$c = $t;"
+        //"$t = abc $z{1,2};"
+        //"$k = a | b | c | d | e | f | g ;"
+        //"$z = q 0% | p 1% | r 1%;"
+        "$x = a ? 0%;"
+        ; // end of string
+//    const char * s = "abczz";
+//
+//
+    LanguageGenerator g;
+    pass = g.parseBNF(def, "$x",TRUE);
+////    LanguageGenerator g(collationBNF, "$root", "$magic", new MagicNode());
+//  
+    if (pass != LanguageGenerator::OK) return FALSE;
+    
+    DUMP_R(TestLanguageGenerator, g, 20);
+    return pass;
+
+    ////UBool pass = strcmp(s,r) == 0;
+
+    //if (pass){
+    //    printf("TestRandomLanguageGenerator passed.\n");
+    //} else {
+    //    printf("TestRandomLanguageGenerator FAILED!!!\n");
+    //}
+    //return pass;
+}
+
+void TestWbnf(void){
+    srand((unsigned)time( NULL ));
+
+    //CALL(TestLiteral);
+    //CALL(TestSequence);
+    //CALL(TestSymbolTable);
+    //CALL(TestVariable);
+
+    //TestRepeat();
+    //TestAlternation();
+    //TestMorph();
+
+    //TestQuote();
+    //TestBuffer();
+    //TestWeightedRand();
+
+    //CALL(TestScanner);
+    //CALL(TestParser);
+    CALL(TestLanguageGenerator);
+}
+