Experimental parser: remove inline code from lexer rules

tools/lexer_generator/code_generator.jinja

 // Copyright 2013 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 48 matching lines @@
       {%- else -%}
         uncompilable code for {{encoding}} {{r[0]}} {{r[1]}}
       {%- endif -%}
     {%- else -%}
       uncompilable code for {{encoding}} {{r[0]}} {{r[1]}}
     {%- endif -%}
   {%- endfor -%}
 {%- endmacro -%}
 
 
-{#- entry actions must not change the value of the cursor -#}
+{#- entry actions must not change the value of the cursor
+    unless they jump -#}
 {% macro dispatch_entry_action(type, value) -%}
-  {%- if type == 'code' %}
-    {#- TODO(dcarney): in debug mode, verify cursor doesn't change -#}
-    {{value}}
-  {% elif type == 'store_token' %}
+  {% if type == 'store_token' %}
     stored_token = Token::{{value}};
   {% elif type == 'set_marker' %}
     marker_ = cursor_ - {{value}};
   {% elif type == 'set_has_escapes' %}
      next_.has_escapes = true;
   {% elif type == 'octal_inside_string' %}
      last_octal_end_ = cursor_;
      next_.has_escapes = true;
   {% elif type == 'line_terminator_in_multiline_comment' %}
      has_multiline_comment_before_next_ = true;
+  {% elif type == 'check_escaped_identifier_start' %}
+    if (!ValidIdentifierStart()) goto default_action;
+    next_.has_escapes = true;
+  {% elif type == 'check_escaped_identifier_part' %}
+    if (!ValidIdentifierPart()) goto default_action;
+    next_.has_escapes = true;
+  {% elif type == 'if_line_terminator_backtrack' %}
+    if (!has_line_terminator_before_next_) {
+      {{dispatch_match_action('backtrack', value)}}
+    }
   {% else %}
-    uncompilable code for {{type}}
+    uncompilable code for {{type}} {{value}}
   {% endif -%}
 {%- endmacro -%}
 
 
 {#- match actions must all explicitly jump or return -#}
 {% macro dispatch_match_action(type, value) -%}
-  {%- if type == 'code' %}
-    {{value}}
-    CRASH()
-  {% elif type == 'terminate' %}
+  {% if type == 'terminate' %}
     DO_EOS();
     return;
   {% elif type == 'terminate_illegal' %}
     start_ = marker_;
     BACKWARD(1);
     DO_TOKEN(Token::ILLEGAL);
     return;
   {% elif type == 'skip' %}
     SKIP();
     goto state_entry_0;
@@ skipping 29 matching lines @@
     if (harmony_{{value[0]}}_) {
       stored_token = Token::{{value[1]}};
     } else {
       stored_token = Token::{{value[2]}};
     }
     goto after_entry_code_{{value[3]}};
   {% elif type == 'octal_number' %}
     last_octal_end_ = cursor_;
     DO_TOKEN(Token::NUMBER);
     return;
+  {% elif type == 'backtrack' %}
+    BACKWARD({{value[0]}});
+    DO_TOKEN(Token::{{value[1]}});
+    return;
   {% else %}
-    uncompilable code for {{encoding}}
+    uncompilable code for {{type}} {{value}}
   {% endif -%}
 {%- endmacro -%}
 
 
 {%- macro long_char_check() -%}
   {%- if encoding == 'utf16'-%}
     primary_char > {{upper_bound}}
   {%- elif encoding == 'utf8'-%}
     primary_char < 0
   {%- else -%}
@@ skipping 99 matching lines @@
           {{ do_transition(transition_state_id) }}
         }
       {% endfor -%}
     }
   {%- endif-%}
 
   {%- set match_action = state.match_action -%}
 
   {%- if match_action %}
     {{ dispatch_match_action(match_action[0], match_action[1]) }}
-    CRASH()
+    CRASH();
   {% else %}
     goto default_action;
   {%- endif %}
 
 {%- endmacro %}
 
 #include "lexer/experimental-scanner.h"
 
 #define PREPARE_TOKEN() {             \
   next_.beg_pos = start_ - buffer_;   \
@@ skipping 61 matching lines @@
     {{ do_dfa_state([dfa_state['node_number']]) }}
   {%- endif -%}
 {%- endfor %}
 
   // Execute the default action.
   default_action:
 {%- if debug_print %}
     fprintf(stderr, "default action\n");
 {% endif -%}
     {{dispatch_match_action(default_action[0], default_action[1])}}
-  CRASH()
+  CRASH();
 
   if (false) {
     // force use of stored_token
     stored_token = Token::ILLEGAL;
     // force use of state_entry_0
     goto state_entry_0;
   }
 }
 } }