Experimental parser: enable transition reduction

tools/lexer_generator/code_generator.jinja

 #include "lexer/experimental-scanner.h"
 
 {%- macro do_key(key) -%}
   {%- for r in key -%}
     {%- if not loop.first %} || {% endif -%}
     {%- if r[0] == 'PRIMARY_RANGE' -%}
       {%- if r[1][0] == r[1][1] -%}
         primary_char == {{r[1][0]}}
       {%- elif r[1][0] == 0 -%}
         primary_char <= {{r[1][1]}}
@@ skipping 42 matching lines @@
   {% elif type == 'skip' %}
     SKIP();
   {% elif type == 'skip_and_terminate' %}
     SKIP();
     --start_;
     {{dispatch_action('terminate', None)}}
   {% elif type == 'line_terminator' %}
     DO_LINE_TERMINATOR();
   {% elif type == 'token' %}
     DO_TOKEN(Token::{{value}})
+  {% elif type == 'goto' %}
+    goto code_after_entry_{{value}};
+  {% elif type == 'store_token' %}
+    stored_token = Token::{{value}};
+  {% elif type == 'do_stored_token' %}
+    DO_TOKEN(stored_token)
   {% elif type == 'do_token_and_go_forward' %}
     DO_TOKEN_AND_GO_FORWARD(Token::{{value}})
   {% elif type == 'harmony_token' %}
     if (harmony_{{value[0]}}_) {
       DO_TOKEN(Token::{{value[1][0]}});
     } else {
       DO_TOKEN(Token::{{value[1][1]}});
     }
+  {% elif type == 'store_harmony_token' %}
+    if (harmony_{{value[0]}}_) {
+      stored_token = Token::{{value[1][0]}};
+    } else {
+      stored_token = Token::{{value[1][1]}};
+    }
   {% elif type == 'set_marker' %}
     marker_ = cursor_ - {{value}};
   {% else %}
     uncompilable code for {{type}}
   {% endif -%}
 {%- endmacro -%}
 
 
 {%- macro long_char_check() -%}
   {%- if encoding == 'utf16'-%}
@@ skipping 42 matching lines @@
 
   {% if debug_print %}
     fprintf(stderr, "state {{state.node_number}}\n");
   {% endif -%}
 
   {%- set entry_action = state.entry_action -%}
   {%- if entry_action %}
     {{ dispatch_action(entry_action[0], entry_action[1]) }}
   {%- endif %}
 
+  {%- if state['has_goto_after_entry'] -%}
+      code_after_entry_{{node_number}}:
+  {%- endif %}
+
   {%- if debug_print %}
     fprintf(stderr, "char at hand is %c (%d)\n", primary_char, primary_char);
   {% endif -%}
 
   {%- macro do_transition(transition_state_id) -%}
     {%- set inline_transition = dfa_states[transition_state_id]['inline'] %}
     FORWARD();
     {%- if inline_transition %}
       {{ do_dfa_state(transition_state_id, True) }}
     {% else %}
@@ skipping 90 matching lines @@
 
 #define SKIP() {                      \
   start_ = cursor_;                   \
 }
 
 namespace v8 {
 namespace internal {
 template<>
 void ExperimentalScanner<{{char_type}}>::Scan() {
   // Setup environment.
+  Token::Value stored_token;
   {{char_type}} primary_char;
   if (cursor_ >= buffer_end_) primary_char = 0;
   else primary_char = *(cursor_);
 
 {# first node is start node #}
 {% for dfa_state in dfa_states -%}
   {%- set inline = dfa_state['inline'] -%}
   {%- if not inline %}
     {{ do_dfa_state(dfa_state['node_number'], False) }}
   {%- endif -%}
 {%- endfor %}
 
   // Should never fall off the edge.
   goto fell_through;
   // Execute the default action.
   default_action:
 {%- if debug_print %}
     fprintf(stderr, "default action\n");
 {% endif -%}
     {{dispatch_action(default_action[0], default_action[1])}}
   FORWARD();
   goto code_start;
 
   fell_through:
-  CHECK(false);
+  ((void(*)())0)(); // crash
+  // force use of stored_token
+  stored_token = Token::ILLEGAL;
 }
 } }