Experimental parser: make string rules look more like ecma spec

src/lexer/lexer_py.re

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 14 matching lines @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 whitespace_char = [ \t\v\f\r:ws:\240];
 whitespace = whitespace_char+;
 identifier_start = [$_a-zA-Z:lit:];
 identifier_char = [0-9:identifier_start:];
 line_terminator = [\n\r];
 digit = [0-9];
 hex_digit = [0-9a-fA-F];
+single_escape_char = ['"\\bfnrtva];
 maybe_exponent = /([eE][\-+]?[:digit:]+)?/;
 number =
   /0[xX][:hex_digit:]+/ | (
   /\.[:digit:]+/ maybe_exponent |
   /[:digit:]+(\.[:digit:]*)?/ maybe_exponent );
+# TODO this is incomplete/incorrect
+line_terminator_sequence = (/\n\r?/)|(/\r\n?/);
 
 # grammar is
 #   regex <action_on_state_entry|action_on_match|transition>
 #
 # actions can be c code enclosed in {} or identifiers to be passed to codegen
 # transition must be in continue or the name of a subgraph
 
 <<default>>
 "|="          <|push_token(ASSIGN_BIT_OR)|>
 "^="          <|push_token(ASSIGN_BIT_XOR)|>
@@ skipping 47 matching lines @@
 "<<="         <|push_token(ASSIGN_SHL)|>
 ">>="         <|push_token(ASSIGN_SAR)|>
 "<="          <|push_token(LTE)|>
 ">="          <|push_token(GTE)|>
 "<<"          <|push_token(SHL)|>
 ">>"          <|push_token(SAR)|>
 "<"           <|push_token(LT)|>
 ">"           <|push_token(GT)|>
 
 number        <|push_token(NUMBER)|>
-# is this necessary?
 number identifier_char   <|push_token(ILLEGAL)|>
 
 "("           <|push_token(LPAREN)|>
 ")"           <|push_token(RPAREN)|>
 "["           <|push_token(LBRACK)|>
 "]"           <|push_token(RBRACK)|>
 "{"           <|push_token(LBRACE)|>
 "}"           <|push_token(RBRACE)|>
 ":"           <|push_token(COLON)|>
 ";"           <|push_token(SEMICOLON)|>
@@ skipping 63 matching lines @@
 "true"        <|push_token(TRUE_LITERAL)|>
 "try"         <|push_token(TRY)|>
 "typeof"      <|push_token(TYPEOF)|>
 "var"         <|push_token(VAR)|>
 "void"        <|push_token(VOID)|>
 "while"       <|push_token(WHILE)|>
 "with"        <|push_token(WITH)|>
 "yield"       <|push_token(YIELD)|>
 
 identifier_start <|push_token(IDENTIFIER)|Identifier>
-/\\u[0-9a-fA-F]{4}/ <{
+/\\u[:hex_digit:]{4}/ <{
   if (V8_UNLIKELY(!ValidIdentifierStart())) {
     goto default_action;
   }
 }|push_token(IDENTIFIER)|Identifier>
 
 eof             <|terminate|>
 default_action  <push_token(ILLEGAL)>
 
 <<DoubleQuoteString>>
-/\\\n\r?/ <||continue>
-/\\\r\n?/ <||continue>
-/\\[xX][:hex_digit:]{2}/ <||continue>
-/\\[^xX\r\n]/     <||continue>
-/\n|\r/   <|push_token(ILLEGAL)|>
-"\""      <|push_token(STRING)|>
-eof       <|terminate_illegal|>
-catch_all <||continue>
+"\\" line_terminator_sequence <||continue>
+/\\[xX][:hex_digit:]{2}/      <||continue>
+/\\[u][:hex_digit:]{4}/       <||continue>
+/\\[^xXu\r\n]/                <||continue>
+"\\"                          <|push_token(ILLEGAL)|>
+/\n|\r/                       <|push_token(ILLEGAL)|>
+"\""                          <|push_token(STRING)|>
+eof                           <|terminate_illegal|>
+catch_all                     <||continue>
 
 <<SingleQuoteString>>
-/\\\n\r?/ <||continue>
-/\\\r\n?/ <||continue>
-/\\[xX][:hex_digit:]{2}/ <||continue>
-/\\[^xX\r\n]/     <||continue>
-/\n|\r/   <|push_token(ILLEGAL)|>
-"'"       <|push_token(STRING)|>
-eof       <|terminate_illegal|>
-catch_all <||continue>
+# TODO subgraph for '\'
+"\\" line_terminator_sequence <||continue>
+/\\[xX][:hex_digit:]{2}/      <||continue>
+/\\[u][:hex_digit:]{4}/       <||continue>
+/\\[^xXu\r\n]/                <||continue>
+"\\"                          <|push_token(ILLEGAL)|>
+/\n|\r/                       <|push_token(ILLEGAL)|>
+"'"                           <|push_token(STRING)|>
+eof                           <|terminate_illegal|>
+catch_all                     <||continue>
 
 <<Identifier>>
 identifier_char <|push_token(IDENTIFIER)|continue>
-/\\u[0-9a-fA-F]{4}/ <{
+/\\u[:hex_digit:]{4}/ <{
   if (V8_UNLIKELY(!ValidIdentifierPart())) {
     goto default_action;
   }
 }|push_token(IDENTIFIER)|continue>
 
 <<SingleLineComment>>
 line_terminator  <|push_line_terminator|>
 eof <|skip_and_terminate|>
 catch_all <||continue>
 
 <<MultiLineComment>>
 /\*+\//             <|skip|>
 # TODO find a way to generate the below rule
 /\*+[^\/*]/       <||continue>
 line_terminator  <push_line_terminator||continue>
 eof <|skip_and_terminate|>
 catch_all        <||continue>