Experimental lexer: track one-byteness on the fly.

tools/lexer_generator/code_generator.jinja

 // Copyright 2013 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 87 matching lines @@
 
 {#- match actions must all explicitly jump or return -#}
 {% macro dispatch_match_action(type, value) -%}
   {% if type == 'terminate' %}
     {{dispatch_match_action('backtrack', ('1', 'EOS'))}}
   {% elif type == 'terminate_illegal' %}
     start_ = marker;
     {{dispatch_match_action('backtrack', ('1', 'ILLEGAL'))}}
   {% elif type == 'skip' %}
     RESET_START();
+    {%- if encoding == 'utf16'-%}
+      next_.is_onebyte = true;
+    {%- endif -%}
     goto state_entry_0;
   {% elif type == 'skip_and_terminate' %}
     RESET_START();
     --start_;
     {{dispatch_match_action('terminate', None)}}
   {% elif type == 'line_terminator' %}
     RESET_START();
+    {%- if encoding == 'utf16'-%}
+      next_.is_onebyte = true;
+    {%- endif -%}
     has_line_terminator_before_next_ = true;
     goto state_entry_0;
   {% elif type == 'token' %}
     DO_TOKEN(Token::{{value}})
     return;
   {% elif type == 'goto_start' %}
     goto state_entry_{{value[0]}};
   {% elif type == 'store_token_and_goto' %}
     stored_token = Token::{{value[0]}};
     goto after_entry_code_{{value[1]}};
@@ skipping 132 matching lines @@
   {% endfor -%}
 
   {%- for key, transition_state_id in state['deferred_transitions'] %}
     if ({{do_key(key)}}) { // deferred transition
       {{ do_transition(transition_state_id) }}
     }
   {% endfor -%}
 
   {%- if state['long_char_transitions'] -%}
     if ({{long_char_check()}}) {
+      next_.is_onebyte = false;
       {{long_char_create()}}
       {%- for key, transition_state_id in state['long_char_transitions'] %}
         if ({{do_key(key)}}) { // long_char transition
           {{ do_transition(transition_state_id) }}
         }
       {% endfor -%}
     }
   {%- endif-%}
 
   {%- set match_action = state.match_action -%}
@@ skipping 39 matching lines @@
 #define CRASH() { }
 #endif
 
 namespace v8 {
 namespace internal {
 template<>
 void ExperimentalScanner<{{char_type}}>::Scan() {
 
   // Setup environment.
   next_.has_escapes = false;
+  next_.is_onebyte = true;
   Token::Value stored_token;
   const {{char_type}} * marker;
   {{char_type}} primary_char;
 
 {# first node is start node #}
 {% for dfa_state in dfa_states -%}
   {%- if not dfa_state['inline'] %}
     {{ do_dfa_state([dfa_state['node_number']]) }}
   {%- endif -%}
 {%- endfor %}
@@ skipping 10 matching lines @@
     // force use of stored_token
     stored_token = Token::ILLEGAL;
     // force use of marker
     marker = NULL;
     // force use of state_entry_0
     goto state_entry_0;
   }
 }
 } }