Experimental parser: add dfa path iterator

src/lexer/lexer_py.re

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 124 matching lines @@
 "-"           <|token(SUB)|>
 "*"           <|token(MUL)|>
 "/"           <|token(DIV)|>
 "%"           <|token(MOD)|>
 "~"           <|token(BIT_NOT)|>
 ","           <|token(COMMA)|>
 
 line_terminator+   <|line_terminator|>
 /[:whitespace:]+/  <|skip|>
 
-"\""           <set_marker(1)||DoubleQuoteString>
-"'"            <set_marker(1)||SingleQuoteString>
+"\""           <set_marker(1)|token(ILLEGAL)|DoubleQuoteString>
+"'"            <set_marker(1)|token(ILLEGAL)|SingleQuoteString>
 
 # all keywords
 "break"       <|token(BREAK)|>
 "case"        <|token(CASE)|>
 "catch"       <|token(CATCH)|>
 "class"       <|token(FUTURE_RESERVED_WORD)|>
 "const"       <|token(CONST)|>
 "continue"    <|token(CONTINUE)|>
 "debugger"    <|token(DEBUGGER)|>
 "default"     <|token(DEFAULT)|>
@@ skipping 28 matching lines @@
 "throw"       <|token(THROW)|>
 "true"        <|token(TRUE_LITERAL)|>
 "try"         <|token(TRY)|>
 "typeof"      <|token(TYPEOF)|>
 "var"         <|token(VAR)|>
 "void"        <|token(VOID)|>
 "while"       <|token(WHILE)|>
 "with"        <|token(WITH)|>
 "yield"       <|token(YIELD)|>
 
-identifier_start <|token(IDENTIFIER)|Identifier>
-unicode_escape <check_escaped_identifier_start|token(IDENTIFIER)|Identifier>
+identifier_start  <|token(IDENTIFIER)|Identifier>
+unicode_escape    <check_escaped_identifier_start|token(IDENTIFIER)|Identifier>
+"\\"              <|token(ILLEGAL)|>  # ambiguous backtracking otherwise
 
 eos             <terminate>
-default_action  <do_token_and_go_forward(ILLEGAL)>
+default_action  <default>
 
 <<DoubleQuoteString>>
 epsilon                       <StringSubgraph>
 "\""                          <|token(STRING)|>
 catch_all                     <||continue>
 eos                           <terminate_illegal>
 
 <<SingleQuoteString>>
 epsilon                       <StringSubgraph>
 "'"                           <|token(STRING)|>
 catch_all                     <||continue>
 eos                           <terminate_illegal>
 
 <<StringSubgraph>>
 "\\" line_terminator_sequence <set_has_escapes||continue(1)>
 /\\[x][:hex_digit:]{2}/       <set_has_escapes||continue(1)>
 unicode_escape                <set_has_escapes||continue(1)>
 /\\[1-7]/                     <octal_inside_string||continue(1)>
 /\\[0-7][0-7]+/               <octal_inside_string||continue(1)>
 "\\0"                         <set_has_escapes||continue(1)>
 /\\[^xu0-7:line_terminator:]/ <set_has_escapes||continue(1)>
 "\\"                          <|token(ILLEGAL)|>
 line_terminator               <|token(ILLEGAL)|>
 
 <<Identifier>>
-identifier_char <|token(IDENTIFIER)|continue>
-/\\u[:hex_digit:]{4}/ <check_escaped_identifier_part|token(IDENTIFIER)|continue>
+identifier_char  <|token(IDENTIFIER)|continue>
+unicode_escape   <check_escaped_identifier_part|token(IDENTIFIER)|continue>
+"\\"             <|token(ILLEGAL)|>  # ambiguous backtracking otherwise
 
 <<SingleLineComment>>
 line_terminator  <|line_terminator|>
 catch_all        <||continue>
 eos              <skip_and_terminate>
 
 <<MultiLineComment>>
 /\*+\//          <|skip|>
-# TODO(dcarney): find a way to generate the below rule
 /\*+[^\/*]/      <||continue>
 line_terminator  <line_terminator_in_multiline_comment||continue>
 catch_all        <||continue>
 eos              <terminate_illegal>