Experimental scanner: number and whitespace fixes.

src/lexer/lexer.re

 // Portions of this code based on re2c:
 //   (re2c/examples/push.re)
 // Copyright 2013 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
@@ skipping 248 matching lines @@
     any = [\000-\377];
     whitespace_char = [ \t\v\f\r];
     whitespace = whitespace_char+;
     identifier_start = [$_a-zA-Z];
     identifier_char = [$_a-zA-Z0-9];
     not_identifier_char = any\identifier_char\[\\];
     line_terminator = [\n\r]+;
     digit = [0-9];
     hex_digit = [0-9a-fA-F];
     maybe_exponent = ('e' [-+]? digit+)?;
-    number = ('0x' hex_digit+) | ("." digit+ maybe_exponent) | (digit+ ("." digit+)? maybe_exponent);
+    number = ('0x' hex_digit+) | ("." digit+ maybe_exponent) | (digit+ ("." digit*)? maybe_exponent);
 
     <Normal> "break" not_identifier_char      { PUSH_TOKEN_LOOKAHEAD(Token::BREAK); }
     <Normal> "case" not_identifier_char       { PUSH_TOKEN_LOOKAHEAD(Token::CASE); }
     <Normal> "catch" not_identifier_char      { PUSH_TOKEN_LOOKAHEAD(Token::CATCH); }
     <Normal> "class" not_identifier_char      { PUSH_TOKEN_LOOKAHEAD(Token::FUTURE_RESERVED_WORD); }
     <Normal> "const" not_identifier_char      { PUSH_TOKEN_LOOKAHEAD(Token::CONST); }
     <Normal> "continue" not_identifier_char   { PUSH_TOKEN_LOOKAHEAD(Token::CONTINUE); }
     <Normal> "debugger" not_identifier_char   { PUSH_TOKEN_LOOKAHEAD(Token::DEBUGGER); }
     <Normal> "default" not_identifier_char    { PUSH_TOKEN_LOOKAHEAD(Token::DEFAULT); }
     <Normal> "delete" not_identifier_char     { PUSH_TOKEN_LOOKAHEAD(Token::DELETE); }
@@ skipping 44 matching lines @@
     <Normal> "%="          { PUSH_TOKEN(Token::ASSIGN_MOD); }
 
     <Normal> "==="         { PUSH_TOKEN(Token::EQ_STRICT); }
     <Normal> "=="          { PUSH_TOKEN(Token::EQ); }
     <Normal> "="           { PUSH_TOKEN(Token::ASSIGN); }
     <Normal> "!=="         { PUSH_TOKEN(Token::NE_STRICT); }
     <Normal> "!="          { PUSH_TOKEN(Token::NE); }
     <Normal> "!"           { PUSH_TOKEN(Token::NOT); }
 
     <Normal> "//"          :=> SingleLineComment
-    <Normal> whitespace? "-->" { if (just_seen_line_terminator_) { YYSETCONDITION(kConditionSingleLineComment); goto yy0; } else { --cursor_; send(Token::DEC); start_ = cursor_; goto yy0; } }
+    <Normal> whitespace* "-->" { if (just_seen_line_terminator_) { YYSETCONDITION(kConditionSingleLineComment); goto yy0; } else { --cursor_; send(Token::DEC); start_ = cursor_; goto yy0; } }
     <Normal> "/*"          :=> MultiLineComment
     <Normal> "<!--"        :=> HtmlComment
 
     <Normal> ">>>="        { PUSH_TOKEN(Token::ASSIGN_SHR); }
     <Normal> ">>>"         { PUSH_TOKEN(Token::SHR); }
     <Normal> "<<="         { PUSH_TOKEN(Token::ASSIGN_SHL); }
     <Normal> ">>="         { PUSH_TOKEN(Token::ASSIGN_SAR); }
     <Normal> "<="          { PUSH_TOKEN(Token::LTE); }
     <Normal> ">="          { PUSH_TOKEN(Token::GTE); }
     <Normal> "<<"          { PUSH_TOKEN(Token::SHL); }
@@ skipping 118 matching lines @@
     size_t start_offset = start_ - buffer_;
     memmove(buffer_, start_, limit_ - start_);
     marker_ -= start_offset;
     cursor_ -= start_offset;
     limit_ -= start_offset;
     start_ -= start_offset;
     real_start_ += start_offset;
   }
   return 0;
 }