Experimental lexer generator: generate code for utf-16 character classes.

tools/lexer_generator/code_generator.jinja

 #include "lexer/even-more-experimental-scanner.h"
 
 {# TODO implement CLASS checks #}
 {%- macro do_key(key) -%}
   {%- for r in key -%}
     {%- if not loop.first %} || {% endif -%}
     {%- if r[0] == 'LATIN_1' -%}
       {%- if r[1][0] == r[1][1] -%}
         yych == {{r[1][0]}}
       {%- elif r[1][0] == 0 -%}
         yych <= {{r[1][1]}}
       {%- elif r[1][1] == 255 -%}
         yych >= {{r[1][0]}}
       {%- else -%}
         ({{r[1][0]}} <= yych && yych <= {{r[1][1]}})
       {%- endif -%}
     {%- elif r[0] == 'CLASS' -%}
       {%- if r[1] == 'eos' -%}
         (yych == 0 && cursor_ >= buffer_end_)
       {%- elif r[1] == 'zero' -%}
         (yych == 0 && cursor_ < buffer_end_)
+      {%- elif r[1] == 'whitespace' and encoding == 'utf16'-%}
+        unicode_cache_->IsWhiteSpace(yych)
+      {%- elif r[1] == 'letter' and encoding == 'utf16'-%}
+        (!(yych >= 'a' && yych <= 'z') && !(yych >= 'A' && yych <= 'Z') &&  unicode_cache_->IsLetter(yych))
+      {%- elif r[1] == 'identifier_part_not_letter' and encoding == 'utf16'-%}
+        unicode_cache_->IsIdentifierPartNotLetter(yych)
       {%- else -%}
-        false
+        false /* {{r[1]}} */
       {%- endif -%}
     {%- else -%}
       false
     {%- endif -%}
   {%- endfor -%}
 {%- endmacro -%}
 
 
 {% macro dispatch_action(type, value) -%}
   {%- if type == 'code' %}
@@ skipping 130 matching lines @@
   else yych = *(cursor_);                 \
 }
 
 #define SKIP() {                      \
   start_ = cursor_;                   \
 }
 
 namespace v8 {
 namespace internal {
 template<>
+{%- if encoding == 'latin1' %}
+  {%- set char_type = 'uint8_t' -%}
+{%- elif encoding == 'utf16' %}
+  {%- set char_type = 'uint16_t' -%}
+{%- endif %}
 Token::Value EvenMoreExperimentalScanner<{{char_type}}>::Next(int* beg_pos_to_return,
                                                               int* end_pos_to_return) {
   // Setup environment.
   {{char_type}} yych;
   if (cursor_ >= buffer_end_) yych = 0;
   else yych = *(cursor_);
 
 {# first node is start node #}
 {% for dfa_state in dfa_states -%}
   {%- set inline = dfa_state['inline'] -%}
@@ skipping 11 matching lines @@
 {% endif -%}
     {{dispatch_action(default_action[0], default_action[1])}}
   FORWARD();
   goto code_start;
 
   fell_through:
   CHECK(false);
 }
 } }