Experimental parser: allocate substrings

tools/lexer_generator/code_generator.jinja

 // Copyright 2013 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 104 matching lines @@
 
 {#- match actions must all explicitly jump or return -#}
 {% macro dispatch_match_action(type, args) -%}
   {% if type == 'terminate' %}
     {{dispatch_match_action('manual_backtrack', ('1', 'EOS'))}}
   {% elif type == 'terminate_illegal' %}
     start_ = marker;
     {{dispatch_match_action('manual_backtrack', ('1', 'ILLEGAL'))}}
   {% elif type == 'skip' %}
     RESET_START();
-    {%- if encoding == 'utf16'-%}
-      next_.is_onebyte = true;
+    {%- if encoding == 'utf16'or encoding == 'utf8' -%}
+      next_.is_in_primary_range = true;
     {%- endif -%}
     goto state_entry_0;
   {% elif type == 'skip_and_terminate' %}
     RESET_START();
     --start_;
     {{dispatch_match_action('terminate', None)}}
   {% elif type == 'line_terminator' %}
     RESET_START();
-    {%- if encoding == 'utf16'-%}
-      next_.is_onebyte = true;
+    {%- if encoding == 'utf16'or encoding == 'utf8' -%}
+      next_.is_in_primary_range = true;
     {%- endif -%}
     has_line_terminator_before_next_ = true;
     goto state_entry_0;
   {% elif type == 'token' %}
     DO_TOKEN(Token::{{args[0]}})
     return;
   {% elif type == 'do_stored_token' %}
     DO_TOKEN(stored_token)
     return;
   {% elif type == 'default' %}
@@ skipping 131 matching lines @@
 
   {%- if 'eos' in state['unique_transitions'] %}
     if (primary_char == 0 && cursor_ >= buffer_end_) { // eos handler
       FORWARD();
       {{ jump(state['unique_transitions']['eos']) }}
     }
   {%- endif -%}
 
   {%- if state['deferred_transitions'] %}
     if ({{long_char_check()}}) {
-      next_.is_onebyte = false;
+      next_.is_in_primary_range = false;
       {{long_char_create()}}
       {%- for key, jump_id in state['deferred_transitions'] %}
         if ({{do_key(key)}}) { // long_char transition
           {{ do_transition(jump_id) }}
         }
       {% endfor -%}
     }
   {%- endif -%}
 
   {%- if state['omega_transition'] %}
@@ skipping 49 matching lines @@
 #endif
 
 
 namespace v8 {
 namespace internal {
 template<>
 void Lexer<{{char_type}}>::Scan() {
 
   // Setup environment.
   next_.has_escapes = false;
-  next_.is_onebyte = true;
+  next_.is_in_primary_range = true;
   Token::Value stored_token;
   const {{char_type}} * marker;
   {{char_type}} primary_char;
   const {{char_type}} * stored_marker;  {# TODO(dcarney): complete this #}
   const {{char_type}} * stored_cursor;
   const {{char_type}} * stored_start;
 
 {# first node is start node #}
 {% for dfa_state in dfa_states -%}
   {%- if not dfa_state['inline'] %}
@@ skipping 14 matching lines @@
     // force use of stored_token
     stored_token = Token::ILLEGAL;
     // force use of marker
     marker = NULL;
     // force use of state_entry_0
     goto state_entry_0;
   }
 }
 } }