Experimental push mode parser: More fixes.

src/lexer/lexer.re

 #include <fcntl.h>
 #include <stdio.h>
 #include <stddef.h>
 #include <stdlib.h>
 #include <string.h>
 
 /*!types:re2c */
 
 #if defined(WIN32)
 
@@ skipping 15 matching lines @@
     #endif
 
 #endif
 
 // ----------------------------------------------------------------------
 #define PUSH_EOS(T) { printf("got eos\n"); }
 #define PUSH_TOKEN(T) { \
         printf("got token %s (%d)\n", tokenNames[T], T); \
          SKIP(); }
 #define PUSH_STRING() { \
-        --cursor; \
         printf("got string\n"); \
         size_t tokenSize = cursor-start; \
         fwrite(start, tokenSize, 1, stdout); \
         printf("\n"); \
         SKIP(); }
 #define PUSH_NUMBER() { \
         --cursor; \
         printf("got number\n"); \
         size_t tokenSize = cursor-start; \
         fwrite(start, tokenSize, 1, stdout); \
@@ skipping 313 matching lines @@
         <Normal> identifier_start        :=> Identifier
         <Normal> number_start            :=> Number
 
         <Normal> eof                     { PUSH_EOS(); return 1; }
         <Normal> any                     { TERMINATE_ILLEGAL(); }
 
         <DoubleQuoteString> "\\\""       { goto yy0; }
         <DoubleQuoteString> '"'          { PUSH_STRING();}
         <DoubleQuoteString> any          { goto yy0; }
 
-        <SingleQuoteString> "\\'"    { goto yy0; }
-        <SingleQuoteString> "'"      { PUSH_STRING();}
-        <SingleQuoteString> any      { goto yy0; }
+        <SingleQuoteString> "\\'"        { goto yy0; }
+        <SingleQuoteString> "'"          { PUSH_STRING();}
+        <SingleQuoteString> any          { goto yy0; }
 
-        <Identifier> identifier_char+  { goto yy0; }
-        <Identifier> any               { PUSH_IDENTIFIER(); }
+        <Identifier> identifier_char+    { goto yy0; }
+        <Identifier> any                 { PUSH_IDENTIFIER(); }
 
         <SingleLineComment> line_terminator { PUSH_LINE_TERMINATOR();}
-        <SingleLineComment> eof { PUSH_LINE_TERMINATOR();}
-        <SingleLineComment> any     :=> SingleLineComment
+        <SingleLineComment> eof          { PUSH_LINE_TERMINATOR();}
+        <SingleLineComment> any          { goto yy0; }
 
-        <MultiLineComment> [*][//]      { PUSH_LINE_TERMINATOR();}
-        <MultiLineComment> eof { TERMINATE_ILLEGAL(); }
-        <MultiLineComment> any       :=> MultiLineComment
+        <MultiLineComment> [*][//]       { PUSH_LINE_TERMINATOR();}
+        <MultiLineComment> eof           { TERMINATE_ILLEGAL(); }
+        <MultiLineComment> any           { goto yy0; }
 
-        <HtmlComment> eof { TERMINATE_ILLEGAL(); }
-        <HtmlComment> "-->"          { PUSH_LINE_TERMINATOR();}
-        <HtmlComment> any            :=> HtmlComment
+        <HtmlComment> eof                { TERMINATE_ILLEGAL(); }
+        <HtmlComment> "-->"              { PUSH_LINE_TERMINATOR();}
+        <HtmlComment> any                { goto yy0; }
 
-        <Number> number_char+          { goto yy0; }
-        <Number> any                   { PUSH_NUMBER(); }
+        <Number> number_char+            { goto yy0; }
+        <Number> any                     { PUSH_NUMBER(); }
 
         */
 
     fill:
         ssize_t unfinishedSize = cursor-start;
         printf(
             "scanner needs a refill. Exiting for now with:\n"
             "    saved fill state = %d\n"
             "    unfinished token size = %ld\n",
             state,
@@ skipping 63 matching lines @@
           return 1;
        }
         if(n<batchSize) break;
     }
     scanner.push(0, -1);
     close(input);
 
     // Done
     return 0;
 }