Experimental parser: enable transition reduction

tools/lexer_generator/code_generator.jinja

Index: tools/lexer_generator/code_generator.jinja
diff --git a/tools/lexer_generator/code_generator.jinja b/tools/lexer_generator/code_generator.jinja
index 68cdf8c42cb35740599d192210fcaae2c29eef34..3b5031623b92a902a3646dc96d3779443eabdfd8 100644
--- a/tools/lexer_generator/code_generator.jinja
+++ b/tools/lexer_generator/code_generator.jinja
@@ -60,6 +60,12 @@
     DO_LINE_TERMINATOR();
   {% elif type == 'token' %}
     DO_TOKEN(Token::{{value}})
+  {% elif type == 'goto' %}
+    goto code_after_entry_{{value}};
+  {% elif type == 'store_token' %}
+    stored_token = Token::{{value}};
+  {% elif type == 'do_stored_token' %}
+    DO_TOKEN(stored_token)
   {% elif type == 'do_token_and_go_forward' %}
     DO_TOKEN_AND_GO_FORWARD(Token::{{value}})
   {% elif type == 'harmony_token' %}
@@ -68,6 +74,12 @@
     } else {
       DO_TOKEN(Token::{{value[1][1]}});
     }
+  {% elif type == 'store_harmony_token' %}
+    if (harmony_{{value[0]}}_) {
+      stored_token = Token::{{value[1][0]}};
+    } else {
+      stored_token = Token::{{value[1][1]}};
+    }
   {% elif type == 'set_marker' %}
     marker_ = cursor_ - {{value}};
   {% else %}
@@ -130,6 +142,10 @@
     {{ dispatch_action(entry_action[0], entry_action[1]) }}
   {%- endif %}
 
+  {%- if state['has_goto_after_entry'] -%}
+      code_after_entry_{{node_number}}:
+  {%- endif %}
+
   {%- if debug_print %}
     fprintf(stderr, "char at hand is %c (%d)\n", primary_char, primary_char);
   {% endif -%}
@@ -240,6 +256,7 @@ namespace internal {
 template<>
 void ExperimentalScanner<{{char_type}}>::Scan() {
   // Setup environment.
+  Token::Value stored_token;
   {{char_type}} primary_char;
   if (cursor_ >= buffer_end_) primary_char = 0;
   else primary_char = *(cursor_);
@@ -264,7 +281,9 @@ void ExperimentalScanner<{{char_type}}>::Scan() {
   goto code_start;
 
   fell_through:
-  CHECK(false);
+  ((void(*)())0)(); // crash
+  // force use of stored_token
+  stored_token = Token::ILLEGAL;
 }
 } }