Add //third_party/sqlite to dirs_to_snapshot, remove net_sql.patch

third_party/sqlite/src/tool/lemon.c

 /*
 ** This file contains all sources (including headers) to the LEMON
 ** LALR(1) parser generator.  The sources have been combined into a
 ** single file to make it easy to include LEMON in the source tree
 ** and Makefile of another program.
 **
 ** The author of this program disclaims copyright.
 */
 #include <stdio.h>
 #include <stdarg.h>
 #include <string.h>
 #include <ctype.h>
 #include <stdlib.h>
 #include <assert.h>
 
 #ifndef __WIN32__
 #   if defined(_WIN32) || defined(WIN32)
-#       define __WIN32__
+#       define __WIN32__
 #   endif
 #endif
 
 #ifdef __WIN32__
 #ifdef __cplusplus
 extern "C" {
 #endif
 extern int access(const char *path, int mode);
 #ifdef __cplusplus
 }
 #endif
 #else
 #include <unistd.h>
 #endif
 
 /* #define PRIVATE static */
 #define PRIVATE
 
 #ifdef TEST
 #define MAXRHS 5       /* Set low to exercise exception code */
 #else
 #define MAXRHS 1000
 #endif
 
 static int showPrecedenceConflict = 0;
-static const char **made_files = NULL;
-static int made_files_count = 0;
-static int successful_exit = 0;
-static void LemonAtExit(void)
-{
-    /* if we failed, delete (most) files we made, to unconfuse build tools. */
-    int i;
-    for (i = 0; i < made_files_count; i++) {
-        if (!successful_exit) {
-            remove(made_files[i]);
-        }
-    }
-    free(made_files);
-    made_files_count = 0;
-    made_files = NULL;
-}
-
 static char *msort(char*,char**,int(*)(const char*,const char*));
 
 /*
 ** Compilers are getting increasingly pedantic about type conversions
 ** as C evolves ever closer to Ada....  To work around the latest problems
 ** we have to define the following variant of strlen().
 */
 #define lemonStrlen(X)   ((int)strlen(X))
 
+/*
+** Compilers are starting to complain about the use of sprintf() and strcpy(),
+** saying they are unsafe.  So we define our own versions of those routines too.
+**
+** There are three routines here:  lemon_sprintf(), lemon_vsprintf(), and
+** lemon_addtext().  The first two are replacements for sprintf() and vsprintf().
+** The third is a helper routine for vsnprintf() that adds texts to the end of a
+** buffer, making sure the buffer is always zero-terminated.
+**
+** The string formatter is a minimal subset of stdlib sprintf() supporting only
+** a few simply conversions:
+**
+**   %d
+**   %s
+**   %.*s
+**
+*/
+static void lemon_addtext(
+  char *zBuf,           /* The buffer to which text is added */
+  int *pnUsed,          /* Slots of the buffer used so far */
+  const char *zIn,      /* Text to add */
+  int nIn,              /* Bytes of text to add.  -1 to use strlen() */
+  int iWidth            /* Field width.  Negative to left justify */
+){
+  if( nIn<0 ) for(nIn=0; zIn[nIn]; nIn++){}
+  while( iWidth>nIn ){ zBuf[(*pnUsed)++] = ' '; iWidth--; }
+  if( nIn==0 ) return;
+  memcpy(&zBuf[*pnUsed], zIn, nIn);
+  *pnUsed += nIn;
+  while( (-iWidth)>nIn ){ zBuf[(*pnUsed)++] = ' '; iWidth++; }
+  zBuf[*pnUsed] = 0;
+}
+static int lemon_vsprintf(char *str, const char *zFormat, va_list ap){
+  int i, j, k, c;
+  int nUsed = 0;
+  const char *z;
+  char zTemp[50];
+  str[0] = 0;
+  for(i=j=0; (c = zFormat[i])!=0; i++){
+    if( c=='%' ){
+      int iWidth = 0;
+      lemon_addtext(str, &nUsed, &zFormat[j], i-j, 0);
+      c = zFormat[++i];
+      if( isdigit(c) || (c=='-' && isdigit(zFormat[i+1])) ){
+        if( c=='-' ) i++;
+        while( isdigit(zFormat[i]) ) iWidth = iWidth*10 + zFormat[i++] - '0';
+        if( c=='-' ) iWidth = -iWidth;
+        c = zFormat[i];
+      }
+      if( c=='d' ){
+        int v = va_arg(ap, int);
+        if( v<0 ){
+          lemon_addtext(str, &nUsed, "-", 1, iWidth);
+          v = -v;
+        }else if( v==0 ){
+          lemon_addtext(str, &nUsed, "0", 1, iWidth);
+        }
+        k = 0;
+        while( v>0 ){
+          k++;
+          zTemp[sizeof(zTemp)-k] = (v%10) + '0';
+          v /= 10;
+        }
+        lemon_addtext(str, &nUsed, &zTemp[sizeof(zTemp)-k], k, iWidth);
+      }else if( c=='s' ){
+        z = va_arg(ap, const char*);
+        lemon_addtext(str, &nUsed, z, -1, iWidth);
+      }else if( c=='.' && memcmp(&zFormat[i], ".*s", 3)==0 ){
+        i += 2;
+        k = va_arg(ap, int);
+        z = va_arg(ap, const char*);
+        lemon_addtext(str, &nUsed, z, k, iWidth);
+      }else if( c=='%' ){
+        lemon_addtext(str, &nUsed, "%", 1, 0);
+      }else{
+        fprintf(stderr, "illegal format\n");
+        exit(1);
+      }
+      j = i+1;
+    }
+  }
+  lemon_addtext(str, &nUsed, &zFormat[j], i-j, 0);
+  return nUsed;
+}
+static int lemon_sprintf(char *str, const char *format, ...){
+  va_list ap;
+  int rc;
+  va_start(ap, format);
+  rc = lemon_vsprintf(str, format, ap);
+  va_end(ap);
+  return rc;
+}
+static void lemon_strcpy(char *dest, const char *src){
+  while( (*(dest++) = *(src++))!=0 ){}
+}
+static void lemon_strcat(char *dest, const char *src){
+  while( *dest ) dest++;
+  lemon_strcpy(dest, src);
+}
+
+
 /* a few forward declarations... */
 struct rule;
 struct lemon;
 struct action;
 
 static struct action *Action_new(void);
 static struct action *Action_sort(struct action *);
 
 /********** From the file "build.h" ************************************/
 void FindRulePrecedences();
@@ skipping 47 matching lines @@
 void ReportOutput(struct lemon *);
 void ReportTable(struct lemon *, int);
 void ReportHeader(struct lemon *);
 void CompressTables(struct lemon *);
 void ResortStates(struct lemon *);
 
 /********** From the file "set.h" ****************************************/
 void  SetSize(int);             /* All sets will be of size N */
 char *SetNew(void);               /* A new set for element 0..N */
 void  SetFree(char*);             /* Deallocate a set */
-
-char *SetNew(void);               /* A new set for element 0..N */
 int SetAdd(char*,int);            /* Add element to a set */
 int SetUnion(char *,char *);    /* A <- A U B, thru element N */
 #define SetFind(X,Y) (X[Y])       /* True if Y is in set X */
 
 /********** From the file "struct.h" *************************************/
 /*
 ** Principal data structures for the LEMON parser generator.
 */
 
 typedef enum {LEMON_FALSE=0, LEMON_TRUE} Boolean;
@@ skipping 516 matching lines @@
         struct symbol *sp = rp->rhs[i];
         if( sp->type==MULTITERMINAL ){
           for(j=0; j<sp->nsubsym; j++){
             if( sp->subsym[j]->prec>=0 ){
               rp->precsym = sp->subsym[j];
               break;
             }
           }
         }else if( sp->prec>=0 ){
           rp->precsym = rp->rhs[i];
-        }
+        }
       }
     }
   }
   return;
 }
 
 /* Find all nonterminals which will generate the empty string.
 ** Then go back and compute the first sets of every nonterminal.
 ** The first set is the set of all terminal symbols which can begin
 ** a string generated by that nonterminal.
@@ skipping 10 matching lines @@
   for(i=lemp->nterminal; i<lemp->nsymbol; i++){
     lemp->symbols[i]->firstset = SetNew();
   }
 
   /* First compute all lambdas */
   do{
     progress = 0;
     for(rp=lemp->rule; rp; rp=rp->next){
       if( rp->lhs->lambda ) continue;
       for(i=0; i<rp->nrhs; i++){
-         struct symbol *sp = rp->rhs[i];
-         if( sp->type!=TERMINAL || sp->lambda==LEMON_FALSE ) break;
+        struct symbol *sp = rp->rhs[i];
+        assert( sp->type==NONTERMINAL || sp->lambda==LEMON_FALSE );
+        if( sp->lambda==LEMON_FALSE ) break;
       }
       if( i==rp->nrhs ){
         rp->lhs->lambda = LEMON_TRUE;
         progress = 1;
       }
     }
   }while( progress );
 
   /* Now compute all first sets */
   do{
     struct symbol *s1, *s2;
     progress = 0;
     for(rp=lemp->rule; rp; rp=rp->next){
       s1 = rp->lhs;
       for(i=0; i<rp->nrhs; i++){
         s2 = rp->rhs[i];
         if( s2->type==TERMINAL ){
           progress += SetAdd(s1->firstset,s2->index);
           break;
         }else if( s2->type==MULTITERMINAL ){
           for(j=0; j<s2->nsubsym; j++){
             progress += SetAdd(s1->firstset,s2->subsym[j]->index);
           }
           break;
-        }else if( s1==s2 ){
+        }else if( s1==s2 ){
           if( s1->lambda==LEMON_FALSE ) break;
-        }else{
+        }else{
           progress += SetUnion(s1->firstset,s2->firstset);
           if( s2->lambda==LEMON_FALSE ) break;
-        }
+        }
       }
     }
   }while( progress );
   return;
 }
 
 /* Compute all LR(0) states for the grammar.  Links
 ** are added to between some states so that the LR(1) follow sets
 ** can be computed later.
 */
@@ skipping 222 matching lines @@
   do{
     progress = 0;
     for(i=0; i<lemp->nstate; i++){
       for(cfp=lemp->sorted[i]->cfp; cfp; cfp=cfp->next){
         if( cfp->status==COMPLETE ) continue;
         for(plp=cfp->fplp; plp; plp=plp->next){
           change = SetUnion(plp->cfp->fws,cfp->fws);
           if( change ){
             plp->cfp->status = INCOMPLETE;
             progress = 1;
-          }
-        }
+          }
+        }
         cfp->status = COMPLETE;
       }
     }
   }while( progress );
 }
 
-static int resolve_conflict(struct action *,struct action *, struct symbol *);
+static int resolve_conflict(struct action *,struct action *);
 
 /* Compute the reduce actions, and resolve conflicts.
 */
 void FindActions(struct lemon *lemp)
 {
   int i,j;
   struct config *cfp;
   struct state *stp;
   struct symbol *sp;
   struct rule *rp;
 
   /* Add all of the reduce actions 
   ** A reduce action is added for each element of the followset of
   ** a configuration which has its dot at the extreme right.
   */
   for(i=0; i<lemp->nstate; i++){   /* Loop over all states */
     stp = lemp->sorted[i];
     for(cfp=stp->cfp; cfp; cfp=cfp->next){  /* Loop over all configurations */
       if( cfp->rp->nrhs==cfp->dot ){        /* Is dot at extreme right? */
         for(j=0; j<lemp->nterminal; j++){
           if( SetFind(cfp->fws,j) ){
             /* Add a reduce action to the state "stp" which will reduce by the
             ** rule "cfp->rp" if the lookahead symbol is "lemp->symbols[j]" */
             Action_add(&stp->ap,REDUCE,lemp->symbols[j],(char *)cfp->rp);
           }
-        }
+        }
       }
     }
   }
 
   /* Add the accepting token */
   if( lemp->start ){
     sp = Symbol_find(lemp->start);
     if( sp==0 ) sp = lemp->rule->lhs;
   }else{
     sp = lemp->rule->lhs;
   }
   /* Add to the first state (which is always the starting state of the
   ** finite state machine) an action to ACCEPT if the lookahead is the
   ** start nonterminal.  */
   Action_add(&lemp->sorted[0]->ap,ACCEPT,sp,0);
 
   /* Resolve conflicts */
   for(i=0; i<lemp->nstate; i++){
     struct action *ap, *nap;
     struct state *stp;
     stp = lemp->sorted[i];
     /* assert( stp->ap ); */
     stp->ap = Action_sort(stp->ap);
     for(ap=stp->ap; ap && ap->next; ap=ap->next){
       for(nap=ap->next; nap && nap->sp==ap->sp; nap=nap->next){
          /* The two actions "ap" and "nap" have the same lookahead.
          ** Figure out which one should be used */
-         lemp->nconflict += resolve_conflict(ap,nap,lemp->errsym);
+         lemp->nconflict += resolve_conflict(ap,nap);
       }
     }
   }
 
   /* Report an error for each rule that can never be reduced. */
   for(rp=lemp->rule; rp; rp=rp->next) rp->canReduce = LEMON_FALSE;
   for(i=0; i<lemp->nstate; i++){
     struct action *ap;
     for(ap=lemp->sorted[i]->ap; ap; ap=ap->next){
       if( ap->type==REDUCE ) ap->x.rp->canReduce = LEMON_TRUE;
@@ skipping 14 matching lines @@
 **   is on an error rule.  In that case, take the action which
 **   is not associated with the error rule.  If neither or both
 **   actions are associated with an error rule, then try to
 **   use precedence to resolve the conflict.
 **
 ** If either action is a SHIFT, then it must be apx.  This
 ** function won't work if apx->type==REDUCE and apy->type==SHIFT.
 */
 static int resolve_conflict(
   struct action *apx,
-  struct action *apy,
-  struct symbol *errsym   /* The error symbol (if defined.  NULL otherwise) */
+  struct action *apy
 ){
   struct symbol *spx, *spy;
   int errcnt = 0;
   assert( apx->sp==apy->sp );  /* Otherwise there would be no conflict */
   if( apx->type==SHIFT && apy->type==SHIFT ){
     apy->type = SSCONFLICT;
     errcnt++;
   }
   if( apx->type==SHIFT && apy->type==REDUCE ){
     spx = apx->sp;
     spy = apy->x.rp->precsym;
     if( spy==0 || spx->prec<0 || spy->prec<0 ){
       /* Not enough precedence information. */
       apy->type = SRCONFLICT;
       errcnt++;
     }else if( spx->prec>spy->prec ){    /* higher precedence wins */
       apy->type = RD_RESOLVED;
     }else if( spx->prec<spy->prec ){
       apx->type = SH_RESOLVED;
     }else if( spx->prec==spy->prec && spx->assoc==RIGHT ){ /* Use operator */
       apy->type = RD_RESOLVED;                             /* associativity */
     }else if( spx->prec==spy->prec && spx->assoc==LEFT ){  /* to break tie */
       apx->type = SH_RESOLVED;
     }else{
       assert( spx->prec==spy->prec && spx->assoc==NONE );
-      apy->type = SRCONFLICT;
-      errcnt++;
+      apx->type = ERROR;
     }
   }else if( apx->type==REDUCE && apy->type==REDUCE ){
     spx = apx->x.rp->precsym;
     spy = apy->x.rp->precsym;
     if( spx==0 || spy==0 || spx->prec<0 ||
     spy->prec<0 || spx->prec==spy->prec ){
       apy->type = RRCONFLICT;
       errcnt++;
     }else if( spx->prec>spy->prec ){
       apy->type = RD_RESOLVED;
@@ skipping 158 matching lines @@
           xsp = rp->rhs[i];
           if( xsp->type==TERMINAL ){
             SetAdd(newcfp->fws,xsp->index);
             break;
           }else if( xsp->type==MULTITERMINAL ){
             int k;
             for(k=0; k<xsp->nsubsym; k++){
               SetAdd(newcfp->fws, xsp->subsym[k]->index);
             }
             break;
-          }else{
+          }else{
             SetUnion(newcfp->fws,xsp->firstset);
             if( xsp->lambda==LEMON_FALSE ) break;
-          }
-        }
+          }
+        }
         if( i==rp->nrhs ) Plink_add(&cfp->fplp,newcfp);
       }
     }
   }
   return;
 }
 
 /* Sort the configuration list */
 void Configlist_sort(){
   current = (struct config *)msort((char *)current,(char **)&(current->next),Configcmp);
@@ skipping 80 matching lines @@
   if( azDefine==0 ){
     fprintf(stderr,"out of memory\n");
     exit(1);
   }
   paz = &azDefine[nDefine-1];
   *paz = (char *) malloc( lemonStrlen(z)+1 );
   if( *paz==0 ){
     fprintf(stderr,"out of memory\n");
     exit(1);
   }
-  strcpy(*paz, z);
+  lemon_strcpy(*paz, z);
   for(z=*paz; *z && *z!='='; z++){}
   *z = 0;
 }
 
 static char *user_templatename = NULL;
 static void handle_T_option(char *z){
   user_templatename = (char *) malloc( lemonStrlen(z)+1 );
   if( user_templatename==0 ){
     memory_error();
   }
-  strcpy(user_templatename, z);
+  lemon_strcpy(user_templatename, z);
 }
 
 /* The main program.  Parse the command line and do it... */
 int main(int argc, char **argv)
 {
   static int version = 0;
   static int rpflag = 0;
   static int basisflag = 0;
   static int compress = 0;
   static int quiet = 0;
@@ skipping 15 matching lines @@
     {OPT_FLAG, "r", (char*)&noResort, "Do not sort or renumber states"},
     {OPT_FLAG, "s", (char*)&statistics,
                                    "Print parser stats to standard output."},
     {OPT_FLAG, "x", (char*)&version, "Print the version number."},
     {OPT_FLAG,0,0,0}
   };
   int i;
   int exitcode;
   struct lemon lem;
 
-  atexit(LemonAtExit);
-
   OptInit(argv,options,stderr);
   if( version ){
      printf("Lemon version 1.0\n");
      exit(0); 
   }
   if( OptNArgs()!=1 ){
     fprintf(stderr,"Exactly one filename argument is required.\n");
     exit(1);
   }
   memset(&lem, 0, sizeof(lem));
@@ skipping 13 matching lines @@
 
   /* Parse the input file */
   Parse(&lem);
   if( lem.errorcnt ) exit(lem.errorcnt);
   if( lem.nrule==0 ){
     fprintf(stderr,"Empty grammar.\n");
     exit(1);
   }
 
   /* Count and index the symbols of the grammar */
+  Symbol_new("{default}");
   lem.nsymbol = Symbol_count();
-  Symbol_new("{default}");
   lem.symbols = Symbol_arrayof();
-  for(i=0; i<=lem.nsymbol; i++) lem.symbols[i]->index = i;
-  qsort(lem.symbols,lem.nsymbol+1,sizeof(struct symbol*), Symbolcmpp);
-  for(i=0; i<=lem.nsymbol; i++) lem.symbols[i]->index = i;
+  for(i=0; i<lem.nsymbol; i++) lem.symbols[i]->index = i;
+  qsort(lem.symbols,lem.nsymbol,sizeof(struct symbol*), Symbolcmpp);
+  for(i=0; i<lem.nsymbol; i++) lem.symbols[i]->index = i;
+  while( lem.symbols[i-1]->type==MULTITERMINAL ){ i--; }
+  assert( strcmp(lem.symbols[i-1]->name,"{default}")==0 );
+  lem.nsymbol = i - 1;
   for(i=1; isupper(lem.symbols[i]->name[0]); i++);
   lem.nterminal = i;
 
   /* Generate a reprint of the grammar, if requested on the command line */
   if( rpflag ){
     Reprint(&lem);
   }else{
     /* Initialize the size for all follow and first sets */
     SetSize(lem.nterminal+1);
 
@@ skipping 43 matching lines @@
       lem.nterminal, lem.nsymbol - lem.nterminal, lem.nrule);
     printf("                   %d states, %d parser table entries, %d conflicts\n",
       lem.nstate, lem.tablesize, lem.nconflict);
   }
   if( lem.nconflict > 0 ){
     fprintf(stderr,"%d parsing conflicts.\n",lem.nconflict);
   }
 
   /* return 0 on success, 1 on failure. */
   exitcode = ((lem.errorcnt > 0) || (lem.nconflict > 0)) ? 1 : 0;
-  successful_exit = (exitcode == 0);
   exit(exitcode);
   return (exitcode);
 }
 /******************** From the file "msort.c" *******************************/
 /*
 ** A generic merge-sort program.
 **
 ** USAGE:
 ** Let "ptr" be a pointer to some structure which is at the head of
 ** a null-terminated list.  Then to sort the list call:
@@ skipping 10 matching lines @@
 ** The function returns a new pointer which is the head of the list
 ** after sorting.
 **
 ** ALGORITHM:
 ** Merge-sort.
 */
 
 /*
 ** Return a pointer to the next structure in the linked list.
 */
-#define NEXT(A) (*(char**)(((unsigned long)A)+offset))
+#define NEXT(A) (*(char**)(((char*)A)+offset))
 
 /*
 ** Inputs:
 **   a:       A sorted, null-terminated linked list.  (May be null).
 **   b:       A sorted, null-terminated linked list.  (May be null).
 **   cmp:     A pointer to the comparison function.
 **   offset:  Offset in the structure to the "next" field.
 **
 ** Return Value:
 **   A pointer to the head of a sorted list containing the elements
@@ skipping 373 matching lines @@
   LHS_ALIAS_3,
   RHS_ALIAS_1,
   RHS_ALIAS_2,
   PRECEDENCE_MARK_1,
   PRECEDENCE_MARK_2,
   RESYNC_AFTER_RULE_ERROR,
   RESYNC_AFTER_DECL_ERROR,
   WAITING_FOR_DESTRUCTOR_SYMBOL,
   WAITING_FOR_DATATYPE_SYMBOL,
   WAITING_FOR_FALLBACK_ID,
-  WAITING_FOR_WILDCARD_ID
+  WAITING_FOR_WILDCARD_ID,
+  WAITING_FOR_CLASS_ID,
+  WAITING_FOR_CLASS_TOKEN
 };
 struct pstate {
   char *filename;       /* Name of the input file */
   int tokenlineno;      /* Linenumber at which current token starts */
   int errorcnt;         /* Number of errors so far */
   char *tokenstart;     /* Text of current token */
   struct lemon *gp;     /* Global state vector */
   enum e_state state;        /* The state of the parser */
   struct symbol *fallback;   /* The fallback token */
+  struct symbol *tkclass;    /* Token class symbol */
   struct symbol *lhs;        /* Left-hand side of current rule */
   const char *lhsalias;      /* Alias for the LHS */
   int nrhs;                  /* Number of right-hand side symbols seen */
   struct symbol *rhs[MAXRHS];  /* RHS symbols */
   const char *alias[MAXRHS]; /* Aliases for each RHS symbol (or NULL) */
   struct rule *prevrule;     /* Previous rule parsed */
   const char *declkeyword;   /* Keyword of a declaration */
   char **declargslot;        /* Where the declaration argument should be put */
   int insertLineMacro;       /* Add #line before declaration insert */
   int *decllinenoslot;       /* Where to write declaration line number */
@@ skipping 23 matching lines @@
       if( x[0]=='%' ){
         psp->state = WAITING_FOR_DECL_KEYWORD;
       }else if( islower(x[0]) ){
         psp->lhs = Symbol_new(x);
         psp->nrhs = 0;
         psp->lhsalias = 0;
         psp->state = WAITING_FOR_ARROW;
       }else if( x[0]=='{' ){
         if( psp->prevrule==0 ){
           ErrorMsg(psp->filename,psp->tokenlineno,
-"There is no prior rule opon which to attach the code \
+"There is no prior rule upon which to attach the code \
 fragment which begins on this line.");
           psp->errorcnt++;
-        }else if( psp->prevrule->code!=0 ){
+        }else if( psp->prevrule->code!=0 ){
           ErrorMsg(psp->filename,psp->tokenlineno,
 "Code fragment beginning on this line is not the first \
 to follow the previous rule.");
           psp->errorcnt++;
         }else{
           psp->prevrule->line = psp->tokenlineno;
           psp->prevrule->code = &x[1];
-        }
+        }
       }else if( x[0]=='[' ){
         psp->state = PRECEDENCE_MARK_1;
       }else{
         ErrorMsg(psp->filename,psp->tokenlineno,
           "Token \"%s\" should be either \"%%\" or a nonterminal name.",
           x);
         psp->errorcnt++;
       }
       break;
     case PRECEDENCE_MARK_1:
@@ skipping 72 matching lines @@
     case IN_RHS:
       if( x[0]=='.' ){
         struct rule *rp;
         rp = (struct rule *)calloc( sizeof(struct rule) + 
              sizeof(struct symbol*)*psp->nrhs + sizeof(char*)*psp->nrhs, 1);
         if( rp==0 ){
           ErrorMsg(psp->filename,psp->tokenlineno,
             "Can't allocate enough memory for this rule.");
           psp->errorcnt++;
           psp->prevrule = 0;
-        }else{
+        }else{
           int i;
           rp->ruleline = psp->tokenlineno;
           rp->rhs = (struct symbol**)&rp[1];
           rp->rhsalias = (const char**)&(rp->rhs[psp->nrhs]);
           for(i=0; i<psp->nrhs; i++){
             rp->rhs[i] = psp->rhs[i];
             rp->rhsalias[i] = psp->alias[i];
-          }
+          }
           rp->lhs = psp->lhs;
           rp->lhsalias = psp->lhsalias;
           rp->nrhs = psp->nrhs;
           rp->code = 0;
           rp->precsym = 0;
           rp->index = psp->gp->nrule++;
           rp->nextlhs = rp->lhs->rule;
           rp->lhs->rule = rp;
           rp->next = 0;
           if( psp->firstrule==0 ){
             psp->firstrule = psp->lastrule = rp;
-          }else{
+          }else{
             psp->lastrule->next = rp;
             psp->lastrule = rp;
-          }
+          }
           psp->prevrule = rp;
-        }
+        }
         psp->state = WAITING_FOR_DECL_OR_RULE;
       }else if( isalpha(x[0]) ){
         if( psp->nrhs>=MAXRHS ){
           ErrorMsg(psp->filename,psp->tokenlineno,
             "Too many symbols on RHS of rule beginning at \"%s\".",
             x);
           psp->errorcnt++;
           psp->state = RESYNC_AFTER_RULE_ERROR;
-        }else{
+        }else{
           psp->rhs[psp->nrhs] = Symbol_new(x);
           psp->alias[psp->nrhs] = 0;
           psp->nrhs++;
-        }
+        }
       }else if( (x[0]=='|' || x[0]=='/') && psp->nrhs>0 ){
         struct symbol *msp = psp->rhs[psp->nrhs-1];
         if( msp->type!=MULTITERMINAL ){
           struct symbol *origsp = msp;
           msp = (struct symbol *) calloc(1,sizeof(*msp));
           memset(msp, 0, sizeof(*msp));
           msp->type = MULTITERMINAL;
           msp->nsubsym = 1;
           msp->subsym = (struct symbol **) calloc(1,sizeof(struct symbol*));
           msp->subsym[0] = origsp;
@@ skipping 43 matching lines @@
     case WAITING_FOR_DECL_KEYWORD:
       if( isalpha(x[0]) ){
         psp->declkeyword = x;
         psp->declargslot = 0;
         psp->decllinenoslot = 0;
         psp->insertLineMacro = 1;
         psp->state = WAITING_FOR_DECL_ARG;
         if( strcmp(x,"name")==0 ){
           psp->declargslot = &(psp->gp->name);
           psp->insertLineMacro = 0;
-        }else if( strcmp(x,"include")==0 ){
+        }else if( strcmp(x,"include")==0 ){
           psp->declargslot = &(psp->gp->include);
-        }else if( strcmp(x,"code")==0 ){
+        }else if( strcmp(x,"code")==0 ){
           psp->declargslot = &(psp->gp->extracode);
-        }else if( strcmp(x,"token_destructor")==0 ){
+        }else if( strcmp(x,"token_destructor")==0 ){
           psp->declargslot = &psp->gp->tokendest;
-        }else if( strcmp(x,"default_destructor")==0 ){
+        }else if( strcmp(x,"default_destructor")==0 ){
           psp->declargslot = &psp->gp->vardest;
-        }else if( strcmp(x,"token_prefix")==0 ){
+        }else if( strcmp(x,"token_prefix")==0 ){
           psp->declargslot = &psp->gp->tokenprefix;
           psp->insertLineMacro = 0;
-        }else if( strcmp(x,"syntax_error")==0 ){
+        }else if( strcmp(x,"syntax_error")==0 ){
           psp->declargslot = &(psp->gp->error);
-        }else if( strcmp(x,"parse_accept")==0 ){
+        }else if( strcmp(x,"parse_accept")==0 ){
           psp->declargslot = &(psp->gp->accept);
-        }else if( strcmp(x,"parse_failure")==0 ){
+        }else if( strcmp(x,"parse_failure")==0 ){
           psp->declargslot = &(psp->gp->failure);
-        }else if( strcmp(x,"stack_overflow")==0 ){
+        }else if( strcmp(x,"stack_overflow")==0 ){
           psp->declargslot = &(psp->gp->overflow);
         }else if( strcmp(x,"extra_argument")==0 ){
           psp->declargslot = &(psp->gp->arg);
           psp->insertLineMacro = 0;
         }else if( strcmp(x,"token_type")==0 ){
           psp->declargslot = &(psp->gp->tokentype);
           psp->insertLineMacro = 0;
         }else if( strcmp(x,"default_type")==0 ){
           psp->declargslot = &(psp->gp->vartype);
           psp->insertLineMacro = 0;
         }else if( strcmp(x,"stack_size")==0 ){
           psp->declargslot = &(psp->gp->stacksize);
           psp->insertLineMacro = 0;
         }else if( strcmp(x,"start_symbol")==0 ){
           psp->declargslot = &(psp->gp->start);
           psp->insertLineMacro = 0;
         }else if( strcmp(x,"left")==0 ){
           psp->preccounter++;
           psp->declassoc = LEFT;
           psp->state = WAITING_FOR_PRECEDENCE_SYMBOL;
         }else if( strcmp(x,"right")==0 ){
           psp->preccounter++;
           psp->declassoc = RIGHT;
           psp->state = WAITING_FOR_PRECEDENCE_SYMBOL;
         }else if( strcmp(x,"nonassoc")==0 ){
           psp->preccounter++;
           psp->declassoc = NONE;
           psp->state = WAITING_FOR_PRECEDENCE_SYMBOL;
-        }else if( strcmp(x,"destructor")==0 ){
+        }else if( strcmp(x,"destructor")==0 ){
           psp->state = WAITING_FOR_DESTRUCTOR_SYMBOL;
-        }else if( strcmp(x,"type")==0 ){
+        }else if( strcmp(x,"type")==0 ){
           psp->state = WAITING_FOR_DATATYPE_SYMBOL;
         }else if( strcmp(x,"fallback")==0 ){
           psp->fallback = 0;
           psp->state = WAITING_FOR_FALLBACK_ID;
         }else if( strcmp(x,"wildcard")==0 ){
           psp->state = WAITING_FOR_WILDCARD_ID;
+        }else if( strcmp(x,"token_class")==0 ){
+          psp->state = WAITING_FOR_CLASS_ID;
         }else{
           ErrorMsg(psp->filename,psp->tokenlineno,
             "Unknown declaration keyword: \"%%%s\".",x);
           psp->errorcnt++;
           psp->state = RESYNC_AFTER_DECL_ERROR;
-        }
+        }
       }else{
         ErrorMsg(psp->filename,psp->tokenlineno,
           "Illegal declaration keyword: \"%s\".",x);
         psp->errorcnt++;
         psp->state = RESYNC_AFTER_DECL_ERROR;
       }
       break;
     case WAITING_FOR_DESTRUCTOR_SYMBOL:
       if( !isalpha(x[0]) ){
         ErrorMsg(psp->filename,psp->tokenlineno,
@@ skipping 34 matching lines @@
     case WAITING_FOR_PRECEDENCE_SYMBOL:
       if( x[0]=='.' ){
         psp->state = WAITING_FOR_DECL_OR_RULE;
       }else if( isupper(x[0]) ){
         struct symbol *sp;
         sp = Symbol_new(x);
         if( sp->prec>=0 ){
           ErrorMsg(psp->filename,psp->tokenlineno,
             "Symbol \"%s\" has already be given a precedence.",x);
           psp->errorcnt++;
-        }else{
+        }else{
           sp->prec = psp->preccounter;
           sp->assoc = psp->declassoc;
-        }
+        }
       }else{
         ErrorMsg(psp->filename,psp->tokenlineno,
           "Can't assign a precedence to \"%s\".",x);
         psp->errorcnt++;
       }
       break;
     case WAITING_FOR_DECL_ARG:
       if( x[0]=='{' || x[0]=='\"' || isalnum(x[0]) ){
         const char *zOld, *zNew;
         char *zBuf, *z;
         int nOld, n, nLine, nNew, nBack;
         int addLineMacro;
         char zLine[50];
         zNew = x;
         if( zNew[0]=='"' || zNew[0]=='{' ) zNew++;
         nNew = lemonStrlen(zNew);
         if( *psp->declargslot ){
           zOld = *psp->declargslot;
         }else{
           zOld = "";
         }
         nOld = lemonStrlen(zOld);
         n = nOld + nNew + 20;
         addLineMacro = !psp->gp->nolinenosflag && psp->insertLineMacro &&
                         (psp->decllinenoslot==0 || psp->decllinenoslot[0]!=0);
         if( addLineMacro ){
           for(z=psp->filename, nBack=0; *z; z++){
             if( *z=='\\' ) nBack++;
           }
-          sprintf(zLine, "#line %d ", psp->tokenlineno);
+          lemon_sprintf(zLine, "#line %d ", psp->tokenlineno);
           nLine = lemonStrlen(zLine);
           n += nLine + lemonStrlen(psp->filename) + nBack;
         }
         *psp->declargslot = (char *) realloc(*psp->declargslot, n);
         zBuf = *psp->declargslot + nOld;
         if( addLineMacro ){
           if( nOld && zBuf[-1]!='\n' ){
             *(zBuf++) = '\n';
           }
           memcpy(zBuf, zLine, nLine);
@@ skipping 54 matching lines @@
         struct symbol *sp = Symbol_new(x);
         if( psp->gp->wildcard==0 ){
           psp->gp->wildcard = sp;
         }else{
           ErrorMsg(psp->filename, psp->tokenlineno,
             "Extra wildcard to token: %s", x);
           psp->errorcnt++;
         }
       }
       break;
+    case WAITING_FOR_CLASS_ID:
+      if( !islower(x[0]) ){
+        ErrorMsg(psp->filename, psp->tokenlineno,
+          "%%token_class must be followed by an identifier: ", x);
+        psp->errorcnt++;
+        psp->state = RESYNC_AFTER_DECL_ERROR;
+     }else if( Symbol_find(x) ){
+        ErrorMsg(psp->filename, psp->tokenlineno,
+          "Symbol \"%s\" already used", x);
+        psp->errorcnt++;
+        psp->state = RESYNC_AFTER_DECL_ERROR;
+      }else{
+        psp->tkclass = Symbol_new(x);
+        psp->tkclass->type = MULTITERMINAL;
+        psp->state = WAITING_FOR_CLASS_TOKEN;
+      }
+      break;
+    case WAITING_FOR_CLASS_TOKEN:
+      if( x[0]=='.' ){
+        psp->state = WAITING_FOR_DECL_OR_RULE;
+      }else if( isupper(x[0]) || ((x[0]=='|' || x[0]=='/') && isupper(x[1])) ){
+        struct symbol *msp = psp->tkclass;
+        msp->nsubsym++;
+        msp->subsym = (struct symbol **) realloc(msp->subsym,
+          sizeof(struct symbol*)*msp->nsubsym);
+        if( !isupper(x[0]) ) x++;
+        msp->subsym[msp->nsubsym-1] = Symbol_new(x);
+      }else{
+        ErrorMsg(psp->filename, psp->tokenlineno,
+          "%%token_class argument \"%s\" should be a token", x);
+        psp->errorcnt++;
+        psp->state = RESYNC_AFTER_DECL_ERROR;
+      }
+      break;
     case RESYNC_AFTER_RULE_ERROR:
 /*      if( x[0]=='.' ) psp->state = WAITING_FOR_DECL_OR_RULE;
 **      break; */
     case RESYNC_AFTER_DECL_ERROR:
       if( x[0]=='.' ) psp->state = WAITING_FOR_DECL_OR_RULE;
       if( x[0]=='%' ) psp->state = WAITING_FOR_DECL_KEYWORD;
       break;
   }
 }
 
@@ skipping 74 matching lines @@
   fp = fopen(ps.filename,"rb");
   if( fp==0 ){
     ErrorMsg(ps.filename,0,"Can't open this file for reading.");
     gp->errorcnt++;
     return;
   }
   fseek(fp,0,2);
   filesize = ftell(fp);
   rewind(fp);
   filebuf = (char *)malloc( filesize+1 );
-  if( filebuf==0 ){
-    ErrorMsg(ps.filename,0,"Can't allocate %d of memory to hold this file.",
-      filesize+1);
+  if( filesize>100000000 || filebuf==0 ){
+    ErrorMsg(ps.filename,0,"Input file too large.");
     gp->errorcnt++;
+    fclose(fp);
     return;
   }
   if( fread(filebuf,1,filesize,fp)!=filesize ){
     ErrorMsg(ps.filename,0,"Can't read in all %d bytes of this file.",
       filesize);
     free(filebuf);
     gp->errorcnt++;
+    fclose(fp);
     return;
   }
   fclose(fp);
   filebuf[filesize] = 0;
 
   /* Make an initial pass through the file to handle %ifdef and %ifndef */
   preprocess_input(filebuf);
 
   /* Now scan the text of the input file */
   lineno = 1;
@@ skipping 38 matching lines @@
         else if( c=='{' ) level++;
         else if( c=='}' ) level--;
         else if( c=='/' && cp[1]=='*' ){  /* Skip comments */
           int prevc;
           cp = &cp[2];
           prevc = 0;
           while( (c= *cp)!=0 && (c!='/' || prevc!='*') ){
             if( c=='\n' ) lineno++;
             prevc = c;
             cp++;
-          }
-        }else if( c=='/' && cp[1]=='/' ){  /* Skip C++ style comments too */
+          }
+        }else if( c=='/' && cp[1]=='/' ){  /* Skip C++ style comments too */
           cp = &cp[2];
           while( (c= *cp)!=0 && c!='\n' ) cp++;
           if( c ) lineno++;
-        }else if( c=='\'' || c=='\"' ){    /* String a character literals */
+        }else if( c=='\'' || c=='\"' ){    /* String a character literals */
           int startchar, prevc;
           startchar = c;
           prevc = 0;
           for(cp++; (c= *cp)!=0 && (c!=startchar || prevc=='\\'); cp++){
             if( c=='\n' ) lineno++;
             if( prevc=='\\' ) prevc = 0;
             else              prevc = c;
-          }
-        }
+          }
+        }
       }
       if( c==0 ){
         ErrorMsg(ps.filename,ps.tokenlineno,
 "C code starting on this line is not terminated before the end of the file.");
         ps.errorcnt++;
         nextcp = cp;
       }else{
         nextcp = cp+1;
       }
     }else if( isalnum(c) ){          /* Identifiers */
@@ skipping 94 matching lines @@
 PRIVATE char *file_makename(struct lemon *lemp, const char *suffix)
 {
   char *name;
   char *cp;
 
   name = (char*)malloc( lemonStrlen(lemp->filename) + lemonStrlen(suffix) + 5 );
   if( name==0 ){
     fprintf(stderr,"Can't allocate space for a filename.\n");
     exit(1);
   }
-  strcpy(name,lemp->filename);
+  lemon_strcpy(name,lemp->filename);
   cp = strrchr(name,'.');
   if( cp ) *cp = 0;
-  strcat(name,suffix);
+  lemon_strcat(name,suffix);
   return name;
 }
 
 /* Open a file with a name based on the name of the input file,
 ** but with a different (specified) suffix, and return a pointer
 ** to the stream */
 PRIVATE FILE *file_open(
   struct lemon *lemp,
   const char *suffix,
   const char *mode
 ){
   FILE *fp;
 
   if( lemp->outname ) free(lemp->outname);
   lemp->outname = file_makename(lemp, suffix);
   fp = fopen(lemp->outname,mode);
   if( fp==0 && *mode=='w' ){
     fprintf(stderr,"Can't open file \"%s\".\n",lemp->outname);
     lemp->errorcnt++;
     return 0;
   }
-
-  /* Add files we create to a list, so we can delete them if we fail. This
-  ** is to keep makefiles from getting confused. We don't include .out files,
-  ** though: this is debug information, and you don't want it deleted if there
-  ** was an error you need to track down.
-  */
-  if(( *mode=='w' ) && (strcmp(suffix, ".out") != 0)){
-    const char **ptr = (const char **)
-        realloc(made_files, sizeof (const char **) * (made_files_count + 1));
-    const char *fname = Strsafe(lemp->outname);
-    if ((ptr == NULL) || (fname == NULL)) {
-        free(ptr);
-        memory_error();
-    }
-    made_files = ptr;
-    made_files[made_files_count++] = fname;
-  }
   return fp;
 }
 
 /* Duplicate the input file without comments and without actions 
 ** on rules */
 void Reprint(struct lemon *lemp)
 {
   struct rule *rp;
   struct symbol *sp;
   int i, j, maxlen, len, ncolumns, skip;
@@ skipping 15 matching lines @@
       printf(" %3d %-*.*s",j,maxlen,maxlen,sp->name);
     }
     printf("\n");
   }
   for(rp=lemp->rule; rp; rp=rp->next){
     printf("%s",rp->lhs->name);
     /*    if( rp->lhsalias ) printf("(%s)",rp->lhsalias); */
     printf(" ::=");
     for(i=0; i<rp->nrhs; i++){
       sp = rp->rhs[i];
-      printf(" %s", sp->name);
       if( sp->type==MULTITERMINAL ){
+        printf(" %s", sp->subsym[0]->name);
         for(j=1; j<sp->nsubsym; j++){
           printf("|%s", sp->subsym[j]->name);
         }
+      }else{
+        printf(" %s", sp->name);
       }
       /* if( rp->rhsalias[i] ) printf("(%s)",rp->rhsalias[i]); */
     }
     printf(".");
     if( rp->precsym ) printf(" [%s]",rp->precsym->name);
     /* if( rp->code ) printf("\n    %s",rp->code); */
     printf("\n");
   }
 }
 
 void ConfigPrint(FILE *fp, struct config *cfp)
 {
   struct rule *rp;
   struct symbol *sp;
   int i, j;
   rp = cfp->rp;
   fprintf(fp,"%s ::=",rp->lhs->name);
   for(i=0; i<=rp->nrhs; i++){
     if( i==cfp->dot ) fprintf(fp," *");
     if( i==rp->nrhs ) break;
     sp = rp->rhs[i];
-    fprintf(fp," %s", sp->name);
     if( sp->type==MULTITERMINAL ){
+      fprintf(fp," %s", sp->subsym[0]->name);
       for(j=1; j<sp->nsubsym; j++){
         fprintf(fp,"|%s",sp->subsym[j]->name);
       }
+    }else{
+      fprintf(fp," %s", sp->name);
     }
   }
 }
 
 /* #define TEST */
 #if 0
 /* Print a set */
 PRIVATE void SetPrint(out,set,lemp)
 FILE *out;
 char *set;
@@ skipping 89 matching lines @@
   fp = file_open(lemp,".out","wb");
   if( fp==0 ) return;
   for(i=0; i<lemp->nstate; i++){
     stp = lemp->sorted[i];
     fprintf(fp,"State %d:\n",stp->statenum);
     if( lemp->basisflag ) cfp=stp->bp;
     else                  cfp=stp->cfp;
     while( cfp ){
       char buf[20];
       if( cfp->dot==cfp->rp->nrhs ){
-        sprintf(buf,"(%d)",cfp->rp->index);
+        lemon_sprintf(buf,"(%d)",cfp->rp->index);
         fprintf(fp,"    %5s ",buf);
       }else{
         fprintf(fp,"          ");
       }
       ConfigPrint(fp,cfp);
       fprintf(fp,"\n");
 #if 0
       SetPrint(fp,cfp->fws,lemp);
       PlinkPrint(fp,cfp->fplp,"To  ");
       PlinkPrint(fp,cfp->bplp,"From");
@@ skipping 44 matching lines @@
 
 #ifdef __WIN32__
   cp = strrchr(argv0,'\\');
 #else
   cp = strrchr(argv0,'/');
 #endif
   if( cp ){
     c = *cp;
     *cp = 0;
     path = (char *)malloc( lemonStrlen(argv0) + lemonStrlen(name) + 2 );
-    if( path ) sprintf(path,"%s/%s",argv0,name);
+    if( path ) lemon_sprintf(path,"%s/%s",argv0,name);
     *cp = c;
   }else{
     pathlist = getenv("PATH");
     if( pathlist==0 ) pathlist = ".:/bin:/usr/bin";
     pathbuf = (char *) malloc( lemonStrlen(pathlist) + 1 );
     path = (char *)malloc( lemonStrlen(pathlist)+lemonStrlen(name)+2 );
     if( (pathbuf != 0) && (path!=0) ){
       pathbufptr = pathbuf;
-      strcpy(pathbuf, pathlist);
+      lemon_strcpy(pathbuf, pathlist);
       while( *pathbuf ){
         cp = strchr(pathbuf,':');
         if( cp==0 ) cp = &pathbuf[lemonStrlen(pathbuf)];
         c = *cp;
         *cp = 0;
-        sprintf(path,"%s/%s",pathbuf,name);
+        lemon_sprintf(path,"%s/%s",pathbuf,name);
         *cp = c;
         if( c==0 ) pathbuf[0] = 0;
         else pathbuf = &cp[1];
         if( access(path,modemask)==0 ) break;
       }
       free(pathbufptr);
     }
   }
   return path;
 }
@@ skipping 70 matching lines @@
     if( in==0 ){
       fprintf(stderr,"Can't open the template file \"%s\".\n",user_templatename);
       lemp->errorcnt++;
       return 0;
     }
     return in;
   }
 
   cp = strrchr(lemp->filename,'.');
   if( cp ){
-    sprintf(buf,"%.*s.lt",(int)(cp-lemp->filename),lemp->filename);
+    lemon_sprintf(buf,"%.*s.lt",(int)(cp-lemp->filename),lemp->filename);
   }else{
-    sprintf(buf,"%s.lt",lemp->filename);
+    lemon_sprintf(buf,"%s.lt",lemp->filename);
   }
   if( access(buf,004)==0 ){
     tpltname = buf;
   }else if( access(templatename,004)==0 ){
     tpltname = templatename;
   }else{
     tpltname = pathsearch(lemp->argv0,templatename,0);
   }
   if( tpltname==0 ){
     fprintf(stderr,"Can't find the parser driver template file \"%s\".\n",
@@ skipping 130 matching lines @@
     n = lemonStrlen(zText);
   }
   if( (int) (n+sizeof(zInt)*2+used) >= alloced ){
     alloced = n + sizeof(zInt)*2 + used + 200;
     z = (char *) realloc(z,  alloced);
   }
   if( z==0 ) return empty;
   while( n-- > 0 ){
     c = *(zText++);
     if( c=='%' && n>0 && zText[0]=='d' ){
-      sprintf(zInt, "%d", p1);
+      lemon_sprintf(zInt, "%d", p1);
       p1 = p2;
-      strcpy(&z[used], zInt);
+      lemon_strcpy(&z[used], zInt);
       used += lemonStrlen(&z[used]);
       zText++;
       n--;
     }else{
       z[used++] = c;
     }
   }
   z[used] = 0;
   return z;
 }
@@ skipping 128 matching lines @@
   struct lemon *lemp,         /* The main info structure for this parser */
   int *plineno,               /* Pointer to the line number */
   int mhflag                  /* True if generating makeheaders output */
 ){
   int lineno = *plineno;    /* The line number of the output */
   char **types;             /* A hash table of datatypes */
   int arraysize;            /* Size of the "types" array */
   int maxdtlength;          /* Maximum length of any ".datatype" field. */
   char *stddt;              /* Standardized name for a datatype */
   int i,j;                  /* Loop counters */
-  int hash;                 /* For hashing the name of a type */
+  unsigned hash;            /* For hashing the name of a type */
   const char *name;         /* Name of the parser */
 
   /* Allocate and initialize types[] and allocate stddt[] */
   arraysize = lemp->nsymbol * 2;
   types = (char**)calloc( arraysize, sizeof(char*) );
+  if( types==0 ){
+    fprintf(stderr,"Out of memory.\n");
+    exit(1);
+  }
   for(i=0; i<arraysize; i++) types[i] = 0;
   maxdtlength = 0;
   if( lemp->vartype ){
     maxdtlength = lemonStrlen(lemp->vartype);
   }
   for(i=0; i<lemp->nsymbol; i++){
     int len;
     struct symbol *sp = lemp->symbols[i];
     if( sp->datatype==0 ) continue;
     len = lemonStrlen(sp->datatype);
     if( len>maxdtlength ) maxdtlength = len;
   }
   stddt = (char*)malloc( maxdtlength*2 + 1 );
-  if( types==0 || stddt==0 ){
+  if( stddt==0 ){
     fprintf(stderr,"Out of memory.\n");
     exit(1);
   }
 
   /* Build a hash table of datatypes. The ".dtnum" field of each symbol
   ** is filled in with the hash index plus 1.  A ".dtnum" value of 0 is
   ** used for terminal symbols.  If there is no %default_type defined then
   ** 0 is also used as the .dtnum value for nonterminals which do not specify
   ** a datatype using the %type directive.
   */
@@ skipping 23 matching lines @@
     for(j=0; stddt[j]; j++){
       hash = hash*53 + stddt[j];
     }
     hash = (hash & 0x7fffffff)%arraysize;
     while( types[hash] ){
       if( strcmp(types[hash],stddt)==0 ){
         sp->dtnum = hash + 1;
         break;
       }
       hash++;
-      if( hash>=arraysize ) hash = 0;
+      if( hash>=(unsigned)arraysize ) hash = 0;
     }
     if( types[hash]==0 ){
       sp->dtnum = hash + 1;
       types[hash] = (char*)malloc( lemonStrlen(stddt)+1 );
       if( types[hash]==0 ){
         fprintf(stderr,"Out of memory.\n");
         exit(1);
       }
-      strcpy(types[hash],stddt);
+      lemon_strcpy(types[hash],stddt);
     }
   }
 
   /* Print out the definition of YYTOKENTYPE and YYMINORTYPE */
   name = lemp->name ? lemp->name : "Parse";
   lineno = *plineno;
   if( mhflag ){ fprintf(out,"#if INTERFACE\n"); lineno++; }
   fprintf(out,"#define %sTOKENTYPE %s\n",name,
     lemp->tokentype?lemp->tokentype:"void*");  lineno++;
   if( mhflag ){ fprintf(out,"#endif\n"); lineno++; }
@@ skipping 65 matching lines @@
 }
 
 /*
 ** Write text on "out" that describes the rule "rp".
 */
 static void writeRuleText(FILE *out, struct rule *rp){
   int j;
   fprintf(out,"%s ::=", rp->lhs->name);
   for(j=0; j<rp->nrhs; j++){
     struct symbol *sp = rp->rhs[j];
-    fprintf(out," %s", sp->name);
-    if( sp->type==MULTITERMINAL ){
+    if( sp->type!=MULTITERMINAL ){
+      fprintf(out," %s", sp->name);
+    }else{
       int k;
+      fprintf(out," %s", sp->subsym[0]->name);
       for(k=1; k<sp->nsubsym; k++){
         fprintf(out,"|%s",sp->subsym[k]->name);
       }
     }
   }
 }
 
 
 /* Generate C source code for the parser */
 void ReportTable(
@@ skipping 280 matching lines @@
           p->name, p->fallback->name);
       }
       lineno++;
     }
   }
   tplt_xfer(lemp->name, in, out, &lineno);
 
   /* Generate a table containing the symbolic name of every symbol
   */
   for(i=0; i<lemp->nsymbol; i++){
-    sprintf(line,"\"%s\",",lemp->symbols[i]->name);
+    lemon_sprintf(line,"\"%s\",",lemp->symbols[i]->name);
     fprintf(out,"  %-15s",line);
     if( (i&3)==3 ){ fprintf(out,"\n"); lineno++; }
   }
   if( (i&3)!=0 ){ fprintf(out,"\n"); lineno++; }
   tplt_xfer(lemp->name,in,out,&lineno);
 
   /* Generate a table containing a text string that describes every
   ** rule in the rule set of the grammar.  This information is used
   ** when tracing REDUCE actions.
   */
@@ skipping 144 matching lines @@
   FILE *out, *in;
   const char *prefix;
   char line[LINESIZE];
   char pattern[LINESIZE];
   int i;
 
   if( lemp->tokenprefix ) prefix = lemp->tokenprefix;
   else                    prefix = "";
   in = file_open(lemp,".h","rb");
   if( in ){
+    int nextChar;
     for(i=1; i<lemp->nterminal && fgets(line,LINESIZE,in); i++){
-      sprintf(pattern,"#define %s%-30s %2d\n",prefix,lemp->symbols[i]->name,i);
+      lemon_sprintf(pattern,"#define %s%-30s %3d\n",
+                    prefix,lemp->symbols[i]->name,i);
       if( strcmp(line,pattern) ) break;
     }
+    nextChar = fgetc(in);
     fclose(in);
-    if( i==lemp->nterminal ){
+    if( i==lemp->nterminal && nextChar==EOF ){
       /* No change in the file.  Don't rewrite it. */
       return;
     }
   }
   out = file_open(lemp,".h","wb");
   if( out ){
     for(i=1; i<lemp->nterminal; i++){
-      fprintf(out,"#define %s%-30s %2d\n",prefix,lemp->symbols[i]->name,i);
+      fprintf(out,"#define %s%-30s %3d\n",prefix,lemp->symbols[i]->name,i);
     }
     fclose(out);  
   }
   return;
 }
 
 /* Reduce the size of the action tables, if possible, by making use
 ** of defaults.
 **
 ** In this version, we take the most frequent REDUCE action and make
@@ skipping 177 matching lines @@
 ** from a specification in the file
 **              "table.q"
 ** by the associative array code building program "aagen".
 ** Do not edit this file!  Instead, edit the specification
 ** file, then rerun aagen.
 */
 /*
 ** Code for processing tables in the LEMON parser generator.
 */
 
-PRIVATE int strhash(const char *x)
+PRIVATE unsigned strhash(const char *x)
 {
-  int h = 0;
-  while( *x) h = h*13 + *(x++);
+  unsigned h = 0;
+  while( *x ) h = h*13 + *(x++);
   return h;
 }
 
 /* Works like strdup, sort of.  Save a string in malloced memory, but
 ** keep strings in a table so that the same string is not in more
 ** than one place.
 */
 const char *Strsafe(const char *y)
 {
   const char *z;
   char *cpy;
 
   if( y==0 ) return 0;
   z = Strsafe_find(y);
   if( z==0 && (cpy=(char *)malloc( lemonStrlen(y)+1 ))!=0 ){
-    strcpy(cpy,y);
+    lemon_strcpy(cpy,y);
     z = cpy;
     Strsafe_insert(z);
   }
   MemoryCheck(z);
   return z;
 }
 
 /* There is one instance of the following structure for each
 ** associative array of type "x1".
 */
@@ skipping 18 matching lines @@
 /* There is only one instance of the array, which is the following */
 static struct s_x1 *x1a;
 
 /* Allocate a new associative array */
 void Strsafe_init(){
   if( x1a ) return;
   x1a = (struct s_x1*)malloc( sizeof(struct s_x1) );
   if( x1a ){
     x1a->size = 1024;
     x1a->count = 0;
-    x1a->tbl = (x1node*)malloc( 
-      (sizeof(x1node) + sizeof(x1node*))*1024 );
+    x1a->tbl = (x1node*)calloc(1024, sizeof(x1node) + sizeof(x1node*));
     if( x1a->tbl==0 ){
       free(x1a);
       x1a = 0;
     }else{
       int i;
       x1a->ht = (x1node**)&(x1a->tbl[1024]);
       for(i=0; i<1024; i++) x1a->ht[i] = 0;
     }
   }
 }
 /* Insert a new record into the array.  Return TRUE if successful.
 ** Prior data with the same key is NOT overwritten */
 int Strsafe_insert(const char *data)
 {
   x1node *np;
-  int h;
-  int ph;
+  unsigned h;
+  unsigned ph;
 
   if( x1a==0 ) return 0;
   ph = strhash(data);
   h = ph & (x1a->size-1);
   np = x1a->ht[h];
   while( np ){
     if( strcmp(np->data,data)==0 ){
       /* An existing entry with the same key is found. */
       /* Fail because overwrite is not allows. */
       return 0;
     }
     np = np->next;
   }
   if( x1a->count>=x1a->size ){
     /* Need to make the hash table bigger */
     int i,size;
     struct s_x1 array;
     array.size = size = x1a->size*2;
     array.count = x1a->count;
-    array.tbl = (x1node*)malloc(
-      (sizeof(x1node) + sizeof(x1node*))*size );
+    array.tbl = (x1node*)calloc(size, sizeof(x1node) + sizeof(x1node*));
     if( array.tbl==0 ) return 0;  /* Fail due to malloc failure */
     array.ht = (x1node**)&(array.tbl[size]);
     for(i=0; i<size; i++) array.ht[i] = 0;
     for(i=0; i<x1a->count; i++){
       x1node *oldnp, *newnp;
       oldnp = &(x1a->tbl[i]);
       h = strhash(oldnp->data) & (size-1);
       newnp = &(array.tbl[i]);
       if( array.ht[h] ) array.ht[h]->from = &(newnp->next);
       newnp->next = array.ht[h];
@@ skipping 12 matching lines @@
   np->next = x1a->ht[h];
   x1a->ht[h] = np;
   np->from = &(x1a->ht[h]);
   return 1;
 }
 
 /* Return a pointer to data assigned to the given key.  Return NULL
 ** if no such key. */
 const char *Strsafe_find(const char *key)
 {
-  int h;
+  unsigned h;
   x1node *np;
 
   if( x1a==0 ) return 0;
   h = strhash(key) & (x1a->size-1);
   np = x1a->ht[h];
   while( np ){
     if( strcmp(np->data,key)==0 ) break;
     np = np->next;
   }
   return np ? np->data : 0;
@@ skipping 21 matching lines @@
     sp->destructor = 0;
     sp->destLineno = 0;
     sp->datatype = 0;
     sp->useCnt = 0;
     Symbol_insert(sp,sp->name);
   }
   sp->useCnt++;
   return sp;
 }
 
-/* Compare two symbols for working purposes
+/* Compare two symbols for sorting purposes.  Return negative,
+** zero, or positive if a is less then, equal to, or greater
+** than b.
 **
 ** Symbols that begin with upper case letters (terminals or tokens)
 ** must sort before symbols that begin with lower case letters
-** (non-terminals).  Other than that, the order does not matter.
+** (non-terminals).  And MULTITERMINAL symbols (created using the
+** %token_class directive) must sort at the very end. Other than
+** that, the order does not matter.
 **
 ** We find experimentally that leaving the symbols in their original
 ** order (the order they appeared in the grammar file) gives the
 ** smallest parser tables in SQLite.
 */
 int Symbolcmpp(const void *_a, const void *_b)
 {
-  const struct symbol **a = (const struct symbol **) _a;
-  const struct symbol **b = (const struct symbol **) _b;
-  int i1 = (**a).index + 10000000*((**a).name[0]>'Z');
-  int i2 = (**b).index + 10000000*((**b).name[0]>'Z');
-  assert( i1!=i2 || strcmp((**a).name,(**b).name)==0 );
-  return i1-i2;
+  const struct symbol *a = *(const struct symbol **) _a;
+  const struct symbol *b = *(const struct symbol **) _b;
+  int i1 = a->type==MULTITERMINAL ? 3 : a->name[0]>'Z' ? 2 : 1;
+  int i2 = b->type==MULTITERMINAL ? 3 : b->name[0]>'Z' ? 2 : 1;
+  return i1==i2 ? a->index - b->index : i1 - i2;
 }
 
 /* There is one instance of the following structure for each
 ** associative array of type "x2".
 */
 struct s_x2 {
   int size;               /* The number of available slots. */
                           /*   Must be a power of 2 greater than or */
                           /*   equal to 1 */
   int count;              /* Number of currently slots filled */
@@ skipping 14 matching lines @@
 /* There is only one instance of the array, which is the following */
 static struct s_x2 *x2a;
 
 /* Allocate a new associative array */
 void Symbol_init(){
   if( x2a ) return;
   x2a = (struct s_x2*)malloc( sizeof(struct s_x2) );
   if( x2a ){
     x2a->size = 128;
     x2a->count = 0;
-    x2a->tbl = (x2node*)malloc( 
-      (sizeof(x2node) + sizeof(x2node*))*128 );
+    x2a->tbl = (x2node*)calloc(128, sizeof(x2node) + sizeof(x2node*));
     if( x2a->tbl==0 ){
       free(x2a);
       x2a = 0;
     }else{
       int i;
       x2a->ht = (x2node**)&(x2a->tbl[128]);
       for(i=0; i<128; i++) x2a->ht[i] = 0;
     }
   }
 }
 /* Insert a new record into the array.  Return TRUE if successful.
 ** Prior data with the same key is NOT overwritten */
 int Symbol_insert(struct symbol *data, const char *key)
 {
   x2node *np;
-  int h;
-  int ph;
+  unsigned h;
+  unsigned ph;
 
   if( x2a==0 ) return 0;
   ph = strhash(key);
   h = ph & (x2a->size-1);
   np = x2a->ht[h];
   while( np ){
     if( strcmp(np->key,key)==0 ){
       /* An existing entry with the same key is found. */
       /* Fail because overwrite is not allows. */
       return 0;
     }
     np = np->next;
   }
   if( x2a->count>=x2a->size ){
     /* Need to make the hash table bigger */
     int i,size;
     struct s_x2 array;
     array.size = size = x2a->size*2;
     array.count = x2a->count;
-    array.tbl = (x2node*)malloc(
-      (sizeof(x2node) + sizeof(x2node*))*size );
+    array.tbl = (x2node*)calloc(size, sizeof(x2node) + sizeof(x2node*));
     if( array.tbl==0 ) return 0;  /* Fail due to malloc failure */
     array.ht = (x2node**)&(array.tbl[size]);
     for(i=0; i<size; i++) array.ht[i] = 0;
     for(i=0; i<x2a->count; i++){
       x2node *oldnp, *newnp;
       oldnp = &(x2a->tbl[i]);
       h = strhash(oldnp->key) & (size-1);
       newnp = &(array.tbl[i]);
       if( array.ht[h] ) array.ht[h]->from = &(newnp->next);
       newnp->next = array.ht[h];
@@ skipping 14 matching lines @@
   np->next = x2a->ht[h];
   x2a->ht[h] = np;
   np->from = &(x2a->ht[h]);
   return 1;
 }
 
 /* Return a pointer to data assigned to the given key.  Return NULL
 ** if no such key. */
 struct symbol *Symbol_find(const char *key)
 {
-  int h;
+  unsigned h;
   x2node *np;
 
   if( x2a==0 ) return 0;
   h = strhash(key) & (x2a->size-1);
   np = x2a->ht[h];
   while( np ){
     if( strcmp(np->key,key)==0 ) break;
     np = np->next;
   }
   return np ? np->data : 0;
@@ skipping 53 matching lines @@
     if( rc==0 ) rc = a->dot - b->dot;
   }
   if( rc==0 ){
     if( a ) rc = 1;
     if( b ) rc = -1;
   }
   return rc;
 }
 
 /* Hash a state */
-PRIVATE int statehash(struct config *a)
+PRIVATE unsigned statehash(struct config *a)
 {
-  int h=0;
+  unsigned h=0;
   while( a ){
     h = h*571 + a->rp->index*37 + a->dot;
     a = a->bp;
   }
   return h;
 }
 
 /* Allocate a new state structure */
 struct state *State_new()
 {
@@ skipping 28 matching lines @@
 /* There is only one instance of the array, which is the following */
 static struct s_x3 *x3a;
 
 /* Allocate a new associative array */
 void State_init(){
   if( x3a ) return;
   x3a = (struct s_x3*)malloc( sizeof(struct s_x3) );
   if( x3a ){
     x3a->size = 128;
     x3a->count = 0;
-    x3a->tbl = (x3node*)malloc( 
-      (sizeof(x3node) + sizeof(x3node*))*128 );
+    x3a->tbl = (x3node*)calloc(128, sizeof(x3node) + sizeof(x3node*));
     if( x3a->tbl==0 ){
       free(x3a);
       x3a = 0;
     }else{
       int i;
       x3a->ht = (x3node**)&(x3a->tbl[128]);
       for(i=0; i<128; i++) x3a->ht[i] = 0;
     }
   }
 }
 /* Insert a new record into the array.  Return TRUE if successful.
 ** Prior data with the same key is NOT overwritten */
 int State_insert(struct state *data, struct config *key)
 {
   x3node *np;
-  int h;
-  int ph;
+  unsigned h;
+  unsigned ph;
 
   if( x3a==0 ) return 0;
   ph = statehash(key);
   h = ph & (x3a->size-1);
   np = x3a->ht[h];
   while( np ){
     if( statecmp(np->key,key)==0 ){
       /* An existing entry with the same key is found. */
       /* Fail because overwrite is not allows. */
       return 0;
     }
     np = np->next;
   }
   if( x3a->count>=x3a->size ){
     /* Need to make the hash table bigger */
     int i,size;
     struct s_x3 array;
     array.size = size = x3a->size*2;
     array.count = x3a->count;
-    array.tbl = (x3node*)malloc(
-      (sizeof(x3node) + sizeof(x3node*))*size );
+    array.tbl = (x3node*)calloc(size, sizeof(x3node) + sizeof(x3node*));
     if( array.tbl==0 ) return 0;  /* Fail due to malloc failure */
     array.ht = (x3node**)&(array.tbl[size]);
     for(i=0; i<size; i++) array.ht[i] = 0;
     for(i=0; i<x3a->count; i++){
       x3node *oldnp, *newnp;
       oldnp = &(x3a->tbl[i]);
       h = statehash(oldnp->key) & (size-1);
       newnp = &(array.tbl[i]);
       if( array.ht[h] ) array.ht[h]->from = &(newnp->next);
       newnp->next = array.ht[h];
@@ skipping 14 matching lines @@
   np->next = x3a->ht[h];
   x3a->ht[h] = np;
   np->from = &(x3a->ht[h]);
   return 1;
 }
 
 /* Return a pointer to data assigned to the given key.  Return NULL
 ** if no such key. */
 struct state *State_find(struct config *key)
 {
-  int h;
+  unsigned h;
   x3node *np;
 
   if( x3a==0 ) return 0;
   h = statehash(key) & (x3a->size-1);
   np = x3a->ht[h];
   while( np ){
     if( statecmp(np->key,key)==0 ) break;
     np = np->next;
   }
   return np ? np->data : 0;
 }
 
 /* Return an array of pointers to all data in the table.
 ** The array is obtained from malloc.  Return NULL if memory allocation
 ** problems, or if the array is empty. */
 struct state **State_arrayof()
 {
   struct state **array;
   int i,size;
   if( x3a==0 ) return 0;
   size = x3a->count;
-  array = (struct state **)malloc( sizeof(struct state *)*size );
+  array = (struct state **)calloc(size, sizeof(struct state *));
   if( array ){
     for(i=0; i<size; i++) array[i] = x3a->tbl[i].data;
   }
   return array;
 }
 
 /* Hash a configuration */
-PRIVATE int confighash(struct config *a)
+PRIVATE unsigned confighash(struct config *a)
 {
-  int h=0;
+  unsigned h=0;
   h = h*571 + a->rp->index*37 + a->dot;
   return h;
 }
 
 /* There is one instance of the following structure for each
 ** associative array of type "x4".
 */
 struct s_x4 {
   int size;               /* The number of available slots. */
                           /*   Must be a power of 2 greater than or */
@@ skipping 15 matching lines @@
 /* There is only one instance of the array, which is the following */
 static struct s_x4 *x4a;
 
 /* Allocate a new associative array */
 void Configtable_init(){
   if( x4a ) return;
   x4a = (struct s_x4*)malloc( sizeof(struct s_x4) );
   if( x4a ){
     x4a->size = 64;
     x4a->count = 0;
-    x4a->tbl = (x4node*)malloc( 
-      (sizeof(x4node) + sizeof(x4node*))*64 );
+    x4a->tbl = (x4node*)calloc(64, sizeof(x4node) + sizeof(x4node*));
     if( x4a->tbl==0 ){
       free(x4a);
       x4a = 0;
     }else{
       int i;
       x4a->ht = (x4node**)&(x4a->tbl[64]);
       for(i=0; i<64; i++) x4a->ht[i] = 0;
     }
   }
 }
 /* Insert a new record into the array.  Return TRUE if successful.
 ** Prior data with the same key is NOT overwritten */
 int Configtable_insert(struct config *data)
 {
   x4node *np;
-  int h;
-  int ph;
+  unsigned h;
+  unsigned ph;
 
   if( x4a==0 ) return 0;
   ph = confighash(data);
   h = ph & (x4a->size-1);
   np = x4a->ht[h];
   while( np ){
     if( Configcmp((const char *) np->data,(const char *) data)==0 ){
       /* An existing entry with the same key is found. */
       /* Fail because overwrite is not allows. */
       return 0;
     }
     np = np->next;
   }
   if( x4a->count>=x4a->size ){
     /* Need to make the hash table bigger */
     int i,size;
     struct s_x4 array;
     array.size = size = x4a->size*2;
     array.count = x4a->count;
-    array.tbl = (x4node*)malloc(
-      (sizeof(x4node) + sizeof(x4node*))*size );
+    array.tbl = (x4node*)calloc(size, sizeof(x4node) + sizeof(x4node*));
     if( array.tbl==0 ) return 0;  /* Fail due to malloc failure */
     array.ht = (x4node**)&(array.tbl[size]);
     for(i=0; i<size; i++) array.ht[i] = 0;
     for(i=0; i<x4a->count; i++){
       x4node *oldnp, *newnp;
       oldnp = &(x4a->tbl[i]);
       h = confighash(oldnp->data) & (size-1);
       newnp = &(array.tbl[i]);
       if( array.ht[h] ) array.ht[h]->from = &(newnp->next);
       newnp->next = array.ht[h];
@@ skipping 36 matching lines @@
 ** as it is removed.  ("f" may be null to avoid this step.) */
 void Configtable_clear(int(*f)(struct config *))
 {
   int i;
   if( x4a==0 || x4a->count==0 ) return;
   if( f ) for(i=0; i<x4a->count; i++) (*f)(x4a->tbl[i].data);
   for(i=0; i<x4a->size; i++) x4a->ht[i] = 0;
   x4a->count = 0;
   return;
 }