Experimental parser: add backtracking

tools/lexer_generator/code_generator.jinja

 // Copyright 2013 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 77 matching lines @@
       stored_token = Token::{{args[2]}};
     }
   {% elif type == 'check_escaped_identifier_start' %}
     if (!ValidIdentifierStart()) goto default_action;
     next_.has_escapes = true;
   {% elif type == 'check_escaped_identifier_part' %}
     if (!ValidIdentifierPart()) goto default_action;
     next_.has_escapes = true;
   {% elif type == 'if_line_terminator_backtrack' %}
     if (!has_line_terminator_before_next_) {
-      {{dispatch_match_action('backtrack', args)}}
+      {{dispatch_match_action('manual_backtrack', args)}}
     }
+  {% elif type == 'store_lexing_state' %}
+    STORE_LEXING_STATE();
+  {% elif type == 'chain' %}
+    {%- for arg in args %}
+      {{dispatch_entry_action(arg.name(), arg.args())}}
+    {% endfor -%}
   {% else %}
     uncompilable code for {{type}} {{args}}
   {% endif -%}
 {%- endmacro -%}
 
 
 {#- match actions must all explicitly jump or return -#}
 {% macro dispatch_match_action(type, args) -%}
   {% if type == 'terminate' %}
-    {{dispatch_match_action('backtrack', ('1', 'EOS'))}}
+    {{dispatch_match_action('manual_backtrack', ('1', 'EOS'))}}
   {% elif type == 'terminate_illegal' %}
     start_ = marker;
-    {{dispatch_match_action('backtrack', ('1', 'ILLEGAL'))}}
+    {{dispatch_match_action('manual_backtrack', ('1', 'ILLEGAL'))}}
   {% elif type == 'skip' %}
     RESET_START();
     {%- if encoding == 'utf16'-%}
       next_.is_onebyte = true;
     {%- endif -%}
     goto state_entry_0;
   {% elif type == 'skip_and_terminate' %}
     RESET_START();
     --start_;
     {{dispatch_match_action('terminate', None)}}
@@ skipping 19 matching lines @@
     if (harmony_{{args[0]}}_) {
       DO_TOKEN(Token::{{args[1]}});
     } else {
       DO_TOKEN(Token::{{args[2]}});
     }
     return;
   {% elif type == 'octal_number' %}
     last_octal_end_ = cursor_;
     DO_TOKEN(Token::NUMBER);
     return;
-  {% elif type == 'backtrack' %}
+  {% elif type == 'manual_backtrack' %}
     BACKWARD({{args[0]}});
     DO_TOKEN(Token::{{args[1]}});
     return;
+  {% elif type == 'backtrack' %}
+    RESTORE_LEXING_STATE();
+    {{dispatch_match_action(args[0].name(), args[0].args())}}
   {% else %}
     uncompilable code for {{type}} {{args}}
   {% endif -%}
 {%- endmacro -%}
 
 
 {%- macro long_char_check() -%}
   {%- if encoding == 'utf16'-%}
     primary_char > {{upper_bound}}
   {%- elif encoding == 'utf8'-%}
@@ skipping 123 matching lines @@
   goto default_action; {# this is wrong, should be CRASH(); #}
 
 {%- endmacro %}
 
 #include "lexer/experimental-scanner.h"
 
 #define RESET_START() {               \
   start_ = cursor_;                   \
 }
 
+#define STORE_LEXING_STATE() {        \
+  stored_marker = marker;             \
+  stored_cursor = cursor_;            \
+  stored_start = start_;              \
+}
+
+#define RESTORE_LEXING_STATE() {      \
+  marker = stored_marker;             \
+  cursor_ = stored_cursor;            \
+  start_ = stored_start;              \
+}
+
 #define DO_TOKEN(T) {                 \
   next_.beg_pos = start_ - buffer_;   \
   next_.end_pos = cursor_ - buffer_;  \
   next_.token = T;                    \
   RESET_START();                      \
 }
 
 #define FORWARD() {                               \
   cursor_++;                                      \
 }
@@ skipping 17 matching lines @@
 namespace internal {
 template<>
 void ExperimentalScanner<{{char_type}}>::Scan() {
 
   // Setup environment.
   next_.has_escapes = false;
   next_.is_onebyte = true;
   Token::Value stored_token;
   const {{char_type}} * marker;
   {{char_type}} primary_char;
+  const {{char_type}} * stored_marker;  {# TODO(dcarney): complete this #}
+  const {{char_type}} * stored_cursor;
+  const {{char_type}} * stored_start;
 
 {# first node is start node #}
 {% for dfa_state in dfa_states -%}
   {%- if not dfa_state['inline'] %}
     {{ do_dfa_state(dfa_state['node_number']) }}
   {%- endif -%}
 {%- endfor %}
 
   // Execute the default action.
   default_action:
 {%- if debug_print %}
     fprintf(stderr, "default action\n");
 {% endif -%}
     {%- set action = default_action.term() -%}
     {{dispatch_match_action(action.name(), action.args())}}
   CRASH();
 
   if (false) {
     // force use of stored_token
     stored_token = Token::ILLEGAL;
     // force use of marker
     marker = NULL;
     // force use of state_entry_0
     goto state_entry_0;
   }
 }
 } }