Experimental push model parser: various fixes.

src/lexer/lexer.re

 #include <fcntl.h>
 #include <stdio.h>
 #include <stddef.h>
 #include <stdlib.h>
 #include <string.h>
 
 /*!types:re2c */
 
 #if defined(WIN32)
 
@@ skipping 11 matching lines @@
     #include <unistd.h>
 
     #ifndef O_BINARY
         #define O_BINARY 0
     #endif
 
 #endif
 
 // ----------------------------------------------------------------------
 #define PUSH_EOS(T) { printf("got eos\n"); }
-#define PUSH_T(T) { printf("got token %d\n", T); SKIP(); }
-#define PUSH_STRING() { printf("got string\n"); SKIP(); }
-#define PUSH_NUMBER() { printf("got number\n"); SKIP(); }
+#define PUSH_TOKEN(T) { \
+        printf("got token %s (%d)\n", tokenNames[T], T); \
+         SKIP(); }
+#define PUSH_STRING() { \
+        --cursor; \
+        printf("got string\n"); \
+        size_t tokenSize = cursor-start; \
+        fwrite(start, tokenSize, 1, stdout); \
+        printf("\n"); \
+        SKIP(); }
+#define PUSH_NUMBER() { \
+        --cursor; \
+        printf("got number\n"); \
+        size_t tokenSize = cursor-start; \
+        fwrite(start, tokenSize, 1, stdout); \
+        printf("\n"); \
+        SKIP(); }
 #define PUSH_IDENTIFIER() { \
+        --cursor; \
         printf("got identifier: "); \
         size_t tokenSize = cursor-start; \
         fwrite(start, tokenSize, 1, stdout); \
         printf("\n"); \
         SKIP(); }
 #define PUSH_LINE_TERMINATOR() { printf("got line terminator\n"); SKIP();}
 #define TERMINATE_ILLEGAL() { return 1; }
 
 #define TOKENS \
         TOK(EOS) \
@@ skipping 198 matching lines @@
         memcpy(limit, input, inputSize);
         limit += inputSize;
 
         // The scanner starts here
         #define YYLIMIT         limit
         #define YYCURSOR        cursor
         #define YYMARKER        marker
         #define YYCTYPE         uint8_t
 
         #define SKIP()          { start = cursor; YYSETCONDITION(EConditionNormal); goto yy0; }
-        #define SEND(x)         { send(x); SKIP();          }
         #define YYFILL(n)       { goto fill;                }
 
         #define YYGETSTATE()    state
         #define YYSETSTATE(x)   { state = (x);  }
 
         #define YYGETCONDITION() condition
         #define YYSETCONDITION(x) { condition = (x);  }
 
     start:
 
         printf("Starting a round; state: %d, condition: %d\n", state, condition);
 
         /*!re2c
         re2c:indent:top      = 1;
         re2c:yych:conversion = 0;
         re2c:condenumprefix          = ECondition;
         re2c:define:YYCONDTYPE       = Condition;
 
         eof = "\000";
         any = [\000-\377];
         whitespace_char = [ \t\v\f\r];
         whitespace = whitespace_char+;
         identifier_start = [$_\\a-zA-z];
         identifier_char = [$_\\a-zA-z0-9];
         number_start = [0-9];
         number_char = [0-9\.e];
         line_terminator = [\n\r]+;
 
-        <Normal> "("                     { PUSH_T(LPAREN); }
-        <Normal> ")"                     { PUSH_T(RPAREN); }
-        <Normal> "["                     { PUSH_T(LBRACK); }
-        <Normal> "]"                     { PUSH_T(RBRACK); }
-        <Normal> "{"                     { PUSH_T(LBRACE); }
-        <Normal> "}"                     { PUSH_T(RBRACE); }
-        <Normal> ":"                     { PUSH_T(COLON); }
-        <Normal> ";"                     { PUSH_T(SEMICOLON); }
-        <Normal> "."                     { PUSH_T(PERIOD); }
-        <Normal> "?"                     { PUSH_T(CONDITIONAL); }
-        <Normal> "++"                    { PUSH_T(INC); }
-        <Normal> "--"                    { PUSH_T(DEC); }
+        <Normal> "("                     { PUSH_TOKEN(LPAREN); }
+        <Normal> ")"                     { PUSH_TOKEN(RPAREN); }
+        <Normal> "["                     { PUSH_TOKEN(LBRACK); }
+        <Normal> "]"                     { PUSH_TOKEN(RBRACK); }
+        <Normal> "{"                     { PUSH_TOKEN(LBRACE); }
+        <Normal> "}"                     { PUSH_TOKEN(RBRACE); }
+        <Normal> ":"                     { PUSH_TOKEN(COLON); }
+        <Normal> ";"                     { PUSH_TOKEN(SEMICOLON); }
+        <Normal> "."                     { PUSH_TOKEN(PERIOD); }
+        <Normal> "?"                     { PUSH_TOKEN(CONDITIONAL); }
+        <Normal> "++"                    { PUSH_TOKEN(INC); }
+        <Normal> "--"                    { PUSH_TOKEN(DEC); }
 
-        <Normal> "|="                    { PUSH_T(ASSIGN_BIT_OR); }
-        <Normal> "^="                    { PUSH_T(ASSIGN_BIT_XOR); }
-        <Normal> "&="                    { PUSH_T(ASSIGN_BIT_AND); }
-        <Normal> "<<="                   { PUSH_T(ASSIGN_SHL); }
-        <Normal> ">>="                   { PUSH_T(ASSIGN_SAR); }
-        <Normal> ">>>="                  { PUSH_T(ASSIGN_SHR); }
-        <Normal> "+="                    { PUSH_T(ASSIGN_ADD); }
-        <Normal> "-="                    { PUSH_T(ASSIGN_SUB); }
-        <Normal> "*="                    { PUSH_T(ASSIGN_MUL); }
-        <Normal> "/="                    { PUSH_T(ASSIGN_DIV); }
-        <Normal> "%="                    { PUSH_T(ASSIGN_MOD); }
+        <Normal> "|="                    { PUSH_TOKEN(ASSIGN_BIT_OR); }
+        <Normal> "^="                    { PUSH_TOKEN(ASSIGN_BIT_XOR); }
+        <Normal> "&="                    { PUSH_TOKEN(ASSIGN_BIT_AND); }
+        <Normal> "<<="                   { PUSH_TOKEN(ASSIGN_SHL); }
+        <Normal> ">>="                   { PUSH_TOKEN(ASSIGN_SAR); }
+        <Normal> ">>>="                  { PUSH_TOKEN(ASSIGN_SHR); }
+        <Normal> "+="                    { PUSH_TOKEN(ASSIGN_ADD); }
+        <Normal> "-="                    { PUSH_TOKEN(ASSIGN_SUB); }
+        <Normal> "*="                    { PUSH_TOKEN(ASSIGN_MUL); }
+        <Normal> "/="                    { PUSH_TOKEN(ASSIGN_DIV); }
+        <Normal> "%="                    { PUSH_TOKEN(ASSIGN_MOD); }
 
-        <Normal> ","                     { PUSH_T(COMMA); }
-        <Normal> "||"                    { PUSH_T(OR); }
-        <Normal> "&&"                    { PUSH_T(AND); }
-        <Normal> "|"                     { PUSH_T(BIT_OR); }
-        <Normal> "^"                     { PUSH_T(BIT_XOR); }
-        <Normal> "&"                     { PUSH_T(BIT_AND); }
-        <Normal> "<<"                    { PUSH_T(SHL); }
-        <Normal> ">>"                    { PUSH_T(SAR); }
-        <Normal> "+"                     { PUSH_T(ADD); }
-        <Normal> "-"                     { PUSH_T(SUB); }
-        <Normal> "*"                     { PUSH_T(MUL); }
-        <Normal> "/"                     { PUSH_T(DIV); }
-        <Normal> "%"                     { PUSH_T(MOD); }
+        <Normal> ","                     { PUSH_TOKEN(COMMA); }
+        <Normal> "||"                    { PUSH_TOKEN(OR); }
+        <Normal> "&&"                    { PUSH_TOKEN(AND); }
+        <Normal> "|"                     { PUSH_TOKEN(BIT_OR); }
+        <Normal> "^"                     { PUSH_TOKEN(BIT_XOR); }
+        <Normal> "&"                     { PUSH_TOKEN(BIT_AND); }
+        <Normal> "<<"                    { PUSH_TOKEN(SHL); }
+        <Normal> ">>"                    { PUSH_TOKEN(SAR); }
+        <Normal> "+"                     { PUSH_TOKEN(ADD); }
+        <Normal> "-"                     { PUSH_TOKEN(SUB); }
+        <Normal> "*"                     { PUSH_TOKEN(MUL); }
+        <Normal> "/"                     { PUSH_TOKEN(DIV); }
+        <Normal> "%"                     { PUSH_TOKEN(MOD); }
 
-        <Normal> "==="                   { PUSH_T(EQ_STRICT); }
-        <Normal> "=="                    { PUSH_T(EQ); }
-        <Normal> "!=="                   { PUSH_T(NE_STRICT); }
-        <Normal> "!="                    { PUSH_T(NE); }
-        <Normal> "<="                    { PUSH_T(LTE); }
-        <Normal> ">="                    { PUSH_T(GTE); }
-        <Normal> "<"                     { PUSH_T(LT); }
-        <Normal> ">"                     { PUSH_T(GT); }
+        <Normal> "==="                   { PUSH_TOKEN(EQ_STRICT); }
+        <Normal> "=="                    { PUSH_TOKEN(EQ); }
+        <Normal> "!=="                   { PUSH_TOKEN(NE_STRICT); }
+        <Normal> "!="                    { PUSH_TOKEN(NE); }
+        <Normal> "<="                    { PUSH_TOKEN(LTE); }
+        <Normal> ">="                    { PUSH_TOKEN(GTE); }
+        <Normal> "<"                     { PUSH_TOKEN(LT); }
+        <Normal> ">"                     { PUSH_TOKEN(GT); }
 
-        <Normal> "="                     { PUSH_T(ASSIGN); }
+        <Normal> "="                     { PUSH_TOKEN(ASSIGN); }
 
-        <Normal> "!"                     { PUSH_T(NOT); }
-        <Normal> "~"                     { PUSH_T(BIT_NOT); }
+        <Normal> "!"                     { PUSH_TOKEN(NOT); }
+        <Normal> "~"                     { PUSH_TOKEN(BIT_NOT); }
 
         <Normal> line_terminator+        { PUSH_LINE_TERMINATOR(); }
         <Normal> whitespace              { SKIP();}
 
         <Normal> "//"                    :=> SingleLineComment
         <Normal> "/*"                    :=> MultiLineComment
         <Normal> "<!--"                  :=> HtmlComment
 
         <Normal> ["]                     :=> DoubleQuoteString
         <Normal> [']                     :=> SingleQuoteString
 
         <Normal> identifier_start        :=> Identifier
         <Normal> number_start            :=> Number
 
         <Normal> eof                     { PUSH_EOS(); return 1; }
         <Normal> any                     { TERMINATE_ILLEGAL(); }
 
-        <DoubleQuoteString> "\\\""       {}
-        <DoubleQuoteString> ["]          { PUSH_STRING();}
-        <DoubleQuoteString> any          {}
+        <DoubleQuoteString> "\\\""       { goto yy0; }
+        <DoubleQuoteString> '"'          { PUSH_STRING();}
+        <DoubleQuoteString> any          { goto yy0; }
 
-        <SingleQuoteString> "\\'"    {}
+        <SingleQuoteString> "\\'"    { goto yy0; }
         <SingleQuoteString> "'"      { PUSH_STRING();}
-        <SingleQuoteString> any      {}
+        <SingleQuoteString> any      { goto yy0; }
 
-        <Identifier> identifier_char+  {}
+        <Identifier> identifier_char+  { goto yy0; }
         <Identifier> any               { PUSH_IDENTIFIER(); }
 
-        <SingleLineComment> line_terminator
-                                       { PUSH_LINE_TERMINATOR();}
-
-        <SingleLineComment> any+     {}
+        <SingleLineComment> line_terminator { PUSH_LINE_TERMINATOR();}
+        <SingleLineComment> eof { PUSH_LINE_TERMINATOR();}
+        <SingleLineComment> any     :=> SingleLineComment
 
         <MultiLineComment> [*][//]      { PUSH_LINE_TERMINATOR();}
         <MultiLineComment> eof { TERMINATE_ILLEGAL(); }
-        <MultiLineComment> any+       {}
+        <MultiLineComment> any       :=> MultiLineComment
 
-        <HtmlComment> any+            {}
         <HtmlComment> eof { TERMINATE_ILLEGAL(); }
-        <HtmlComment> "-->"           { }
+        <HtmlComment> "-->"          { PUSH_LINE_TERMINATOR();}
+        <HtmlComment> any            :=> HtmlComment
 
-        <Number> number_char+          { }
+        <Number> number_char+          { goto yy0; }
         <Number> any                   { PUSH_NUMBER(); }
 
         */
 
     fill:
         ssize_t unfinishedSize = cursor-start;
         printf(
             "scanner needs a refill. Exiting for now with:\n"
             "    saved fill state = %d\n"
             "    unfinished token size = %ld\n",
@@ skipping 64 matching lines @@
           return 1;
        }
         if(n<batchSize) break;
     }
     scanner.push(0, -1);
     close(input);
 
     // Done
     return 0;
 }