Experimental parser: utf8 added to build

tools/lexer_generator/code_generator.jinja

 #include "lexer/experimental-scanner.h"
 
 {%- macro do_key(key) -%}
   {%- for r in key -%}
     {%- if not loop.first %} || {% endif -%}
     {%- if r[0] == 'PRIMARY_RANGE' -%}
       {%- if r[1][0] == r[1][1] -%}
         primary_char == {{r[1][0]}}
       {%- elif r[1][0] == 0 -%}
         primary_char <= {{r[1][1]}}
@@ skipping 56 matching lines @@
       PUSH_TOKEN(Token::{{value[1][1]}});
     }
   {% elif type == 'set_marker' %}
     marker_ = cursor_ - {{value}};
   {% else %}
     uncompilable code for {{type}}
   {% endif -%}
 {%- endmacro -%}
 
 
+{%- macro long_char_check() -%}
+  {%- if encoding == 'utf16'-%}
+    primary_char > {{upper_bound}}
+  {%- elif encoding == 'utf8'-%}
+    primary_char < 0
+  {%- else -%}
+    uncompilable code for {{encoding}}
+  {%- endif -%}
+{%- endmacro -%}
+
+
+{%- macro long_char_create() -%}
+  {%- if encoding == 'utf16'-%}
+    const uint32_t long_char = primary_char;
+  {%- elif encoding == 'utf8'-%}
+    unsigned bytes_read = 0;
+    const uint32_t long_char = unibrow::Utf8::CalculateValue(
+        reinterpret_cast<uint8_t*>(cursor_),
+        buffer_end_ - cursor_,
+        &bytes_read);
+    cursor_ += bytes_read;
+    if (long_char == unibrow::Utf8::kBadChar) goto default_action;
+  {%- else -%}
+    uncompilable code for {{encoding}}
+  {%- endif -%}
+{%- endmacro -%}
+
+
 {%- macro do_dfa_state(node_number, inline) -%}
 
   {%- set state = dfa_states[node_number] -%}
 
   {%- if not inline -%}
     {%- if state['inline'] -%}
       bad generated code for {{node_number}}
     {%- endif -%}
     {%- if start_node_number == node_number -%}
       code_start:
@@ skipping 46 matching lines @@
     }
   {% endfor -%}
 
   {%- for key, transition_state_id in state['deferred_transitions'] %}
     if ({{do_key(key)}}) { // deferred transition
       {{ do_transition(transition_state_id) }}
     }
   {% endfor -%}
 
   {%- if state['long_char_transitions'] -%}
-    {# TODO macro this up for utf8 #}
-    if (primary_char > {{upper_bound}}) {
-      uint32_t long_char = primary_char;
+    if ({{long_char_check()}}) {
+      {{long_char_create()}}
       {%- for key, transition_state_id in state['long_char_transitions'] %}
         if ({{do_key(key)}}) { // long_char transition
           {{ do_transition(transition_state_id) }}
         }
       {% endfor -%}
     }
   {%- endif-%}
 
   {%- set match_action = state.match_action -%}
 
@@ skipping 76 matching lines @@
 {% endif -%}
     {{dispatch_action(default_action[0], default_action[1])}}
   FORWARD();
   goto code_start;
 
   fell_through:
   CHECK(false);
 }
 } }