Experimental lexer generator: generate code for utf-16 character classes.

tools/lexer_generator/code_generator.jinja

Index: tools/lexer_generator/code_generator.jinja
diff --git a/tools/lexer_generator/code_generator.jinja b/tools/lexer_generator/code_generator.jinja
index 9db497c4d9f1aeea42d422ea30e24b67bfed72a8..0e5fb129825c92169f8dccd8f07ac4a9237df239 100644
--- a/tools/lexer_generator/code_generator.jinja
+++ b/tools/lexer_generator/code_generator.jinja
@@ -19,8 +19,14 @@
 	(yych == 0 && cursor_ >= buffer_end_)
       {%- elif r[1] == 'zero' -%}
 	(yych == 0 && cursor_ < buffer_end_)
+      {%- elif r[1] == 'whitespace' and encoding == 'utf16'-%}
+        unicode_cache_->IsWhiteSpace(yych)
+      {%- elif r[1] == 'letter' and encoding == 'utf16'-%}
+        (!(yych >= 'a' && yych <= 'z') && !(yych >= 'A' && yych <= 'Z') &&  unicode_cache_->IsLetter(yych))
+      {%- elif r[1] == 'identifier_part_not_letter' and encoding == 'utf16'-%}
+        unicode_cache_->IsIdentifierPartNotLetter(yych)
       {%- else -%}
-	false
+	false /* {{r[1]}} */
       {%- endif -%}
     {%- else -%}
       false
@@ -171,6 +177,11 @@
 namespace v8 {
 namespace internal {
 template<>
+{%- if encoding == 'latin1' %}
+  {%- set char_type = 'uint8_t' -%}
+{%- elif encoding == 'utf16' %}
+  {%- set char_type = 'uint16_t' -%}
+{%- endif %}
 Token::Value EvenMoreExperimentalScanner<{{char_type}}>::Next(int* beg_pos_to_return,
                                                               int* end_pos_to_return) {
   // Setup environment.