Experimental parser: support subgraph inlining

src/lexer/lexer_py.re

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 14 matching lines @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 # Character classes and auxiliary regexps.
 
 line_terminator = [:line_terminator:];
 identifier_start = [$_:letter:];
 identifier_char = [:identifier_start::identifier_part_not_letter:];
 digit = [0-9];
 hex_digit = [0-9a-fA-F];
+unicode_escape = /\\u[:hex_digit:]{4}/;
 single_escape_char = ['"\\bfnrtv];
 maybe_exponent = /([eE][\-+]?[:digit:]+)?/;
 octal_number = /0[0-7]+/;
 
 # Octal numbers are pretty complicated. For example, 01.0 is invalid, since it's
 # parsed as 2 numbers (01 and .0), since 01 is a valid octal number. However,
 # 09.0 is a valid, since 09 cannot be octal. In addition, 0 and 0.1 are valid
 # numbers starting with 0.
 
 non_octal_whole_part = "0" | (
@@ skipping 142 matching lines @@
 "true"        <|token(TRUE_LITERAL)|>
 "try"         <|token(TRY)|>
 "typeof"      <|token(TYPEOF)|>
 "var"         <|token(VAR)|>
 "void"        <|token(VOID)|>
 "while"       <|token(WHILE)|>
 "with"        <|token(WITH)|>
 "yield"       <|token(YIELD)|>
 
 identifier_start <|token(IDENTIFIER)|Identifier>
-/\\u[:hex_digit:]{4}/ <check_escaped_identifier_start|token(IDENTIFIER)|Identifier>
+unicode_escape <check_escaped_identifier_start|token(IDENTIFIER)|Identifier>
 
 eos             <terminate>
 default_action  <do_token_and_go_forward(ILLEGAL)>
 
 <<DoubleQuoteString>>
-"\\" line_terminator_sequence <set_has_escapes||continue>
-/\\[x][:hex_digit:]{2}/       <set_has_escapes||continue>
-/\\[u][:hex_digit:]{4}/       <set_has_escapes||continue>
-/\\[1-7]/                     <octal_inside_string||continue>
-/\\[0-7][0-7]+/               <octal_inside_string||continue>
-"\\0"                         <set_has_escapes||continue>
-/\\[^xu0-7:line_terminator:]/ <set_has_escapes||continue>
-"\\"                          <|token(ILLEGAL)|>
-line_terminator               <|token(ILLEGAL)|>
+epsilon                       <StringSubgraph>
 "\""                          <|token(STRING)|>
 catch_all                     <||continue>
 eos                           <terminate_illegal>
 
 <<SingleQuoteString>>
-# TODO subgraph for '\'
-"\\" line_terminator_sequence <set_has_escapes||continue>
-/\\[x][:hex_digit:]{2}/       <set_has_escapes||continue>
-/\\[u][:hex_digit:]{4}/       <set_has_escapes||continue>
-/\\[1-7]/                     <octal_inside_string||continue>
-/\\[0-7][0-7]+/               <octal_inside_string||continue>
-"\\0"                         <set_has_escapes||continue>
-/\\[^xu0-7:line_terminator:]/ <set_has_escapes||continue>
-"\\"                          <|token(ILLEGAL)|>
-line_terminator               <|token(ILLEGAL)|>
+epsilon                       <StringSubgraph>
 "'"                           <|token(STRING)|>
 catch_all                     <||continue>
 eos                           <terminate_illegal>
 
+<<StringSubgraph>>
+"\\" line_terminator_sequence <set_has_escapes||continue(1)>
+/\\[x][:hex_digit:]{2}/       <set_has_escapes||continue(1)>
+unicode_escape                <set_has_escapes||continue(1)>
+/\\[1-7]/                     <octal_inside_string||continue(1)>
+/\\[0-7][0-7]+/               <octal_inside_string||continue(1)>
+"\\0"                         <set_has_escapes||continue(1)>
+/\\[^xu0-7:line_terminator:]/ <set_has_escapes||continue(1)>
+"\\"                          <|token(ILLEGAL)|>
+line_terminator               <|token(ILLEGAL)|>
+
 <<Identifier>>
 identifier_char <|token(IDENTIFIER)|continue>
 /\\u[:hex_digit:]{4}/ <check_escaped_identifier_part|token(IDENTIFIER)|continue>
 
 <<SingleLineComment>>
 line_terminator  <|line_terminator|>
+catch_all        <||continue>
 eos              <skip_and_terminate>
-catch_all        <||continue>
 
 <<MultiLineComment>>
 /\*+\//          <|skip|>
-# TODO find a way to generate the below rule
+# TODO(dcarney): find a way to generate the below rule
 /\*+[^\/*]/      <||continue>
 line_terminator  <line_terminator_in_multiline_comment||continue>
 catch_all        <||continue>
 eos              <terminate_illegal>