Experimental parser: parsing regex subexpressions

src/lexer/lexer_py.re

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
 #       with the distribution.
 #     * Neither the name of Google Inc. nor the names of its
 #       contributors may be used to endorse or promote products derived
 #       from this software without specific prior written permission.
 #
 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-whitespace_char = [ \t\v\f\r\240:ws:];
+whitespace_char = [ \t\v\f\r:ws:]; # TODO put back \240
 whitespace = whitespace_char+;
 identifier_start = [$_a-zA-Z:lit:];
 identifier_char = [$_a-zA-Z0-9:lit:];
 not_identifier_char = [^:identifier_char:];
 line_terminator = [\n\r]+;
 digit = [0-9];
 hex_digit = [0-9a-fA-F];
-maybe_exponent = ("e" [-+]? digit+)?;
+maybe_exponent = ("e" [\-+]? digit+)?;
 number = ("0x" hex_digit+) | (("." digit+ maybe_exponent) | (digit+ ("." digit*)? maybe_exponent));
 
 <Normal> "break" not_identifier_char      { PUSH_TOKEN_LOOKAHEAD(Token::BREAK); }
 <Normal> "case" not_identifier_char       { PUSH_TOKEN_LOOKAHEAD(Token::CASE); }
 <Normal> "catch" not_identifier_char      { PUSH_TOKEN_LOOKAHEAD(Token::CATCH); }
 <Normal> "class" not_identifier_char      { PUSH_TOKEN_LOOKAHEAD(Token::FUTURE_RESERVED_WORD); }
 <Normal> "const" not_identifier_char      { PUSH_TOKEN_LOOKAHEAD(Token::CONST); }
 <Normal> "continue" not_identifier_char   { PUSH_TOKEN_LOOKAHEAD(Token::CONTINUE); }
 <Normal> "debugger" not_identifier_char   { PUSH_TOKEN_LOOKAHEAD(Token::DEBUGGER); }
 <Normal> "default" not_identifier_char    { PUSH_TOKEN_LOOKAHEAD(Token::DEFAULT); }
@@ skipping 97 matching lines @@
 <Normal> "~"           { PUSH_TOKEN(Token::BIT_NOT); }
 <Normal> ","           { PUSH_TOKEN(Token::COMMA); }
 
 <Normal> line_terminator  { PUSH_LINE_TERMINATOR(); }
 <Normal> whitespace       { SKIP(); }
 
 <Normal> "\""           :=> DoubleQuoteString
 <Normal> "'"           :=> SingleQuoteString
 
 <Normal> identifier_start     :=> Identifier
-<Normal> "\\u[0-9a-fA-F]{4}" { if (ValidIdentifierStart()) { YYSETCONDITION(kConditionIdentifier); goto yyc_Identifier; } send(Token::ILLEGAL); start_ = cursor_; goto yyc_Normal; }
+<Normal> /\\u[0-9a-fA-F]{4}/ { if (ValidIdentifierStart()) { YYSETCONDITION(kConditionIdentifier); goto yyc_Identifier; } send(Token::ILLEGAL); start_ = cursor_; goto yyc_Normal; }
 <Normal> "\\"                 { PUSH_TOKEN(Token::ILLEGAL); }
 
 <Normal> eof           { PUSH_EOF_AND_RETURN();}
 <Normal> any           { PUSH_TOKEN(Token::ILLEGAL); }
 
 <DoubleQuoteString> "\\\\"  { goto yyc_DoubleQuoteString; }
 <DoubleQuoteString> "\\\""  { goto yyc_DoubleQuoteString; }
 <DoubleQuoteString> "\""     { PUSH_TOKEN(Token::STRING);}
-<DoubleQuoteString> "\\\n\r?" { goto yyc_DoubleQuoteString; }
-<DoubleQuoteString> "\\\r\n?" { goto yyc_DoubleQuoteString; }
+<DoubleQuoteString> /\\\n\r?/ { goto yyc_DoubleQuoteString; }
+<DoubleQuoteString> /\\\r\n?/ { goto yyc_DoubleQuoteString; }
 <DoubleQuoteString> "\n"    => Normal { PUSH_TOKEN_LOOKAHEAD(Token::ILLEGAL); }
 <DoubleQuoteString> "\r"    => Normal { PUSH_TOKEN_LOOKAHEAD(Token::ILLEGAL); }
 <DoubleQuoteString> eof     { TERMINATE_ILLEGAL(); }
 <DoubleQuoteString> any     { goto yyc_DoubleQuoteString; }
 
-<SingleQuoteString> "\\\\"  { goto yyc_SingleQuoteString; }
+<SingleQuoteString> "\\"  { goto yyc_SingleQuoteString; }
 <SingleQuoteString> "\\'"   { goto yyc_SingleQuoteString; }
-<SingleQuoteString> "'"     { PUSH_TOKEN(Token::STRING);}
-<SingleQuoteString> "\\\n\r?" { goto yyc_SingleQuoteString; }
-<SingleQuoteString> "\\\r\n?" { goto yyc_SingleQuoteString; }
+<SingleQuoteString> "'"     { PUSH_TOKEN(Token::STRING); }
+<SingleQuoteString> /\\\n\r?/ { goto yyc_SingleQuoteString; }
+<SingleQuoteString> /\\\r\n?/ { goto yyc_SingleQuoteString; }
 <SingleQuoteString> "\n"    => Normal { PUSH_TOKEN_LOOKAHEAD(Token::ILLEGAL); }
 <SingleQuoteString> "\r"    => Normal { PUSH_TOKEN_LOOKAHEAD(Token::ILLEGAL); }
 <SingleQuoteString> eof     { TERMINATE_ILLEGAL(); }
 <SingleQuoteString> any     { goto yyc_SingleQuoteString; }
 
 <Identifier> identifier_char+  { goto yyc_Identifier; }
-<Identifier> "\\u[0-9a-fA-F]{4}" { if (ValidIdentifierPart()) { goto yyc_Identifier; } YYSETCONDITION(kConditionNormal); send(Token::ILLEGAL); start_ = cursor_; goto yyc_Normal; }
+<Identifier> /\\u[0-9a-fA-F]{4}/ { if (ValidIdentifierPart()) { goto yyc_Identifier; } YYSETCONDITION(kConditionNormal); send(Token::ILLEGAL); start_ = cursor_; goto yyc_Normal; }
 <Identifier> "\\"              { PUSH_TOKEN(Token::ILLEGAL); }
 <Identifier> any               { PUSH_TOKEN_LOOKAHEAD(Token::IDENTIFIER); }
 
 <SingleLineComment> line_terminator { PUSH_LINE_TERMINATOR();}
 <SingleLineComment> eof             { start_ = cursor_ - 1; PUSH_TOKEN(Token::EOS); }
 <SingleLineComment> any             { goto yyc_SingleLineComment; }
 
-<MultiLineComment> "*//"  { PUSH_LINE_TERMINATOR();}
+<MultiLineComment> "*/"  { PUSH_LINE_TERMINATOR();}
 <MultiLineComment> eof      { start_ = cursor_ - 1; PUSH_TOKEN(Token::EOS); }
 <MultiLineComment> any      { goto yyc_MultiLineComment; }
 
-<HtmlComment> eof        { start_ = cursor_ - 1; PUSH_TOKEN(Token::EOS); }
 <HtmlComment> "-->"      { PUSH_LINE_TERMINATOR();}
 <HtmlComment> line_terminator+ { PUSH_LINE_TERMINATOR();}
+<HtmlComment> eof        { start_ = cursor_ - 1; PUSH_TOKEN(Token::EOS); }
 <HtmlComment> any        { goto yyc_HtmlComment; }