Experimental push model parser: Numbers.

src/lexer/lexer.re

 #include <fcntl.h>
 #include <stdio.h>
 #include <stddef.h>
 #include <stdlib.h>
 #include <string.h>
 
+
+/*
+TODO:
+- SpiderMonkey compatibility hack: "    --> something" is treated as a single line comment.
+- An identifier cannot start immediately after a number.
+
+*/
+
+
 /*!types:re2c */
 
 #if defined(WIN32)
 
     typedef signed char     int8_t;
     typedef signed short    int16_t;
     typedef signed int      int32_t;
 
     typedef unsigned char   uint8_t;
     typedef unsigned short  uint16_t;
@@ skipping 15 matching lines @@
 #define PUSH_TOKEN(T) { \
         printf("got token %s (%d)\n", tokenNames[T], T); \
          SKIP(); }
 #define PUSH_STRING() { \
         printf("got string\n"); \
         size_t tokenSize = cursor-start; \
         fwrite(start, tokenSize, 1, stdout); \
         printf("\n"); \
         SKIP(); }
 #define PUSH_NUMBER() { \
-        --cursor; \
         printf("got number\n"); \
         size_t tokenSize = cursor-start; \
         fwrite(start, tokenSize, 1, stdout); \
         printf("\n"); \
         SKIP(); }
 #define PUSH_IDENTIFIER() { \
         --cursor; \
         printf("got identifier: "); \
         size_t tokenSize = cursor-start; \
         fwrite(start, tokenSize, 1, stdout); \
@@ skipping 229 matching lines @@
         re2c:yych:conversion = 0;
         re2c:condenumprefix          = ECondition;
         re2c:define:YYCONDTYPE       = Condition;
 
         eof = "\000";
         any = [\000-\377];
         whitespace_char = [ \t\v\f\r];
         whitespace = whitespace_char+;
         identifier_start = [$_\\a-zA-z];
         identifier_char = [$_\\a-zA-z0-9];
-        number_start = [0-9];
-        number_char = [0-9\.e];
         line_terminator = [\n\r]+;
+        digit = [0-9];
+        hex_digit = [0-9a-fA-F];
+        maybe_exponent = ('e' [-+]? digit+)?;
+
+        <Normal> "|="                    { PUSH_TOKEN(ASSIGN_BIT_OR); }
+        <Normal> "^="                    { PUSH_TOKEN(ASSIGN_BIT_XOR); }
+        <Normal> "&="                    { PUSH_TOKEN(ASSIGN_BIT_AND); }
+        <Normal> "+="                    { PUSH_TOKEN(ASSIGN_ADD); }
+        <Normal> "-="                    { PUSH_TOKEN(ASSIGN_SUB); }
+        <Normal> "*="                    { PUSH_TOKEN(ASSIGN_MUL); }
+        <Normal> "/="                    { PUSH_TOKEN(ASSIGN_DIV); }
+        <Normal> "%="                    { PUSH_TOKEN(ASSIGN_MOD); }
+
+        <Normal> "==="                   { PUSH_TOKEN(EQ_STRICT); }
+        <Normal> "=="                    { PUSH_TOKEN(EQ); }
+        <Normal> "="                     { PUSH_TOKEN(ASSIGN); }
+        <Normal> "!=="                   { PUSH_TOKEN(NE_STRICT); }
+        <Normal> "!="                    { PUSH_TOKEN(NE); }
+        <Normal> "!"                     { PUSH_TOKEN(NOT); }
+
+        <Normal> "//"                    :=> SingleLineComment
+        <Normal> "/*"                    :=> MultiLineComment
+        <Normal> "<!--"                  :=> HtmlComment
+
+        <Normal> ">>>="                  { PUSH_TOKEN(ASSIGN_SHR); }
+        <Normal> "<<="                   { PUSH_TOKEN(ASSIGN_SHL); }
+        <Normal> ">>="                   { PUSH_TOKEN(ASSIGN_SAR); }
+        <Normal> "<="                    { PUSH_TOKEN(LTE); }
+        <Normal> ">="                    { PUSH_TOKEN(GTE); }
+        <Normal> "<<"                    { PUSH_TOKEN(SHL); }
+        <Normal> ">>"                    { PUSH_TOKEN(SAR); }
+        <Normal> "<"                     { PUSH_TOKEN(LT); }
+        <Normal> ">"                     { PUSH_TOKEN(GT); }
+
+        <Normal> '0x' hex_digit+         { PUSH_NUMBER(); }
+        <Normal> "." digit+ maybe_exponent { PUSH_NUMBER(); }
+        <Normal> digit+ ("." digit+)? maybe_exponent { PUSH_NUMBER(); }
 
         <Normal> "("                     { PUSH_TOKEN(LPAREN); }
         <Normal> ")"                     { PUSH_TOKEN(RPAREN); }
         <Normal> "["                     { PUSH_TOKEN(LBRACK); }
         <Normal> "]"                     { PUSH_TOKEN(RBRACK); }
         <Normal> "{"                     { PUSH_TOKEN(LBRACE); }
         <Normal> "}"                     { PUSH_TOKEN(RBRACE); }
         <Normal> ":"                     { PUSH_TOKEN(COLON); }
         <Normal> ";"                     { PUSH_TOKEN(SEMICOLON); }
         <Normal> "."                     { PUSH_TOKEN(PERIOD); }
         <Normal> "?"                     { PUSH_TOKEN(CONDITIONAL); }
         <Normal> "++"                    { PUSH_TOKEN(INC); }
         <Normal> "--"                    { PUSH_TOKEN(DEC); }
 
-        <Normal> "|="                    { PUSH_TOKEN(ASSIGN_BIT_OR); }
-        <Normal> "^="                    { PUSH_TOKEN(ASSIGN_BIT_XOR); }
-        <Normal> "&="                    { PUSH_TOKEN(ASSIGN_BIT_AND); }
-        <Normal> "<<="                   { PUSH_TOKEN(ASSIGN_SHL); }
-        <Normal> ">>="                   { PUSH_TOKEN(ASSIGN_SAR); }
-        <Normal> ">>>="                  { PUSH_TOKEN(ASSIGN_SHR); }
-        <Normal> "+="                    { PUSH_TOKEN(ASSIGN_ADD); }
-        <Normal> "-="                    { PUSH_TOKEN(ASSIGN_SUB); }
-        <Normal> "*="                    { PUSH_TOKEN(ASSIGN_MUL); }
-        <Normal> "/="                    { PUSH_TOKEN(ASSIGN_DIV); }
-        <Normal> "%="                    { PUSH_TOKEN(ASSIGN_MOD); }
-
-        <Normal> ","                     { PUSH_TOKEN(COMMA); }
         <Normal> "||"                    { PUSH_TOKEN(OR); }
         <Normal> "&&"                    { PUSH_TOKEN(AND); }
+
         <Normal> "|"                     { PUSH_TOKEN(BIT_OR); }
         <Normal> "^"                     { PUSH_TOKEN(BIT_XOR); }
         <Normal> "&"                     { PUSH_TOKEN(BIT_AND); }
-        <Normal> "<<"                    { PUSH_TOKEN(SHL); }
-        <Normal> ">>"                    { PUSH_TOKEN(SAR); }
         <Normal> "+"                     { PUSH_TOKEN(ADD); }
         <Normal> "-"                     { PUSH_TOKEN(SUB); }
         <Normal> "*"                     { PUSH_TOKEN(MUL); }
         <Normal> "/"                     { PUSH_TOKEN(DIV); }
         <Normal> "%"                     { PUSH_TOKEN(MOD); }
-
-        <Normal> "==="                   { PUSH_TOKEN(EQ_STRICT); }
-        <Normal> "=="                    { PUSH_TOKEN(EQ); }
-        <Normal> "!=="                   { PUSH_TOKEN(NE_STRICT); }
-        <Normal> "!="                    { PUSH_TOKEN(NE); }
-        <Normal> "<="                    { PUSH_TOKEN(LTE); }
-        <Normal> ">="                    { PUSH_TOKEN(GTE); }
-        <Normal> "<"                     { PUSH_TOKEN(LT); }
-        <Normal> ">"                     { PUSH_TOKEN(GT); }
-
-        <Normal> "="                     { PUSH_TOKEN(ASSIGN); }
-
-        <Normal> "!"                     { PUSH_TOKEN(NOT); }
         <Normal> "~"                     { PUSH_TOKEN(BIT_NOT); }
+        <Normal> ","                     { PUSH_TOKEN(COMMA); }
 
         <Normal> line_terminator+        { PUSH_LINE_TERMINATOR(); }
-        <Normal> whitespace              { SKIP();}
-
-        <Normal> "//"                    :=> SingleLineComment
-        <Normal> "/*"                    :=> MultiLineComment
-        <Normal> "<!--"                  :=> HtmlComment
+        <Normal> whitespace              { SKIP(); }
 
         <Normal> ["]                     :=> DoubleQuoteString
         <Normal> [']                     :=> SingleQuoteString
 
         <Normal> identifier_start        :=> Identifier
-        <Normal> number_start            :=> Number
 
         <Normal> eof                     { PUSH_EOS(); return 1; }
         <Normal> any                     { TERMINATE_ILLEGAL(); }
 
         <DoubleQuoteString> "\\\""       { goto yy0; }
         <DoubleQuoteString> '"'          { PUSH_STRING();}
         <DoubleQuoteString> any          { goto yy0; }
 
         <SingleQuoteString> "\\'"        { goto yy0; }
         <SingleQuoteString> "'"          { PUSH_STRING();}
         <SingleQuoteString> any          { goto yy0; }
 
         <Identifier> identifier_char+    { goto yy0; }
         <Identifier> any                 { PUSH_IDENTIFIER(); }
 
         <SingleLineComment> line_terminator { PUSH_LINE_TERMINATOR();}
         <SingleLineComment> eof          { PUSH_LINE_TERMINATOR();}
         <SingleLineComment> any          { goto yy0; }
 
         <MultiLineComment> [*][//]       { PUSH_LINE_TERMINATOR();}
         <MultiLineComment> eof           { TERMINATE_ILLEGAL(); }
         <MultiLineComment> any           { goto yy0; }
 
         <HtmlComment> eof                { TERMINATE_ILLEGAL(); }
         <HtmlComment> "-->"              { PUSH_LINE_TERMINATOR();}
         <HtmlComment> any                { goto yy0; }
-
-        <Number> number_char+            { goto yy0; }
-        <Number> any                     { PUSH_NUMBER(); }
-
         */
 
     fill:
         ssize_t unfinishedSize = cursor-start;
         printf(
             "scanner needs a refill. Exiting for now with:\n"
             "    saved fill state = %d\n"
             "    unfinished token size = %ld\n",
             state,
             unfinishedSize
@@ skipping 62 matching lines @@
           return 1;
        }
         if(n<batchSize) break;
     }
     scanner.push(0, -1);
     close(input);
 
     // Done
     return 0;
 }