Experimental lexer generator: Fix byte order mark detection.

tools/lexer_generator/code_generator.jinja

 #include "lexer/even-more-experimental-scanner.h"
 
 {%- macro do_key(key) -%}
   {%- for r in key -%}
     {%- if not loop.first %} || {% endif -%}
     {%- if r[0] == 'LATIN_1' -%}
       {%- if r[1][0] == r[1][1] -%}
               yych == {{r[1][0]}}
       {%- elif r[1][0] == 0 -%}
               yych <= {{r[1][1]}}
       {%- elif r[1][1] == 255 -%}
               yych >= {{r[1][0]}}
       {%- else -%}
               ({{r[1][0]}} <= yych && yych <= {{r[1][1]}})
       {%- endif -%}
     {%- elif r[0] == 'CLASS' -%}
       {%- if r[1] == 'eos' -%}
               (yych == 0 && cursor_ >= buffer_end_)
       {%- elif r[1] == 'zero' -%}
               (yych == 0 && cursor_ < buffer_end_)
+      {%- elif r[1] == 'byte_order_mark' and encoding == 'utf16'-%}
+        (yych == 0xfffe || yych == 0xfeff)
       {%- elif r[1] == 'non_latin_1_whitespace' and encoding == 'utf16'-%}
         {# FIXME: Add and use unicode_cache_->InNonAsciiWhitespace #}
         (yych > 255 && unicode_cache_->IsWhiteSpace(yych))
       {%- elif r[1] == 'non_latin_1_letter' and encoding == 'utf16'-%}
         {# FIXME: Add and use unicode_cache_->InNonAsciiLetter #}
         (yych > 255 &&  unicode_cache_->IsLetter(yych))
       {%- elif r[1] == 'non_latin1_identifier_part_not_letter' and encoding == 'utf16'-%}
         (yych > 255 &&  unicode_cache_->IsIdentifierPartNotLetter(yych))
       {%- elif r[1] == 'non_latin1_line_terminator' and encoding == 'utf16'-%}
         (yych > 255 &&  unicode_cache_->IsLineTerminator(yych))
@@ skipping 173 matching lines @@
 {% endif -%}
     {{dispatch_action(default_action[0], default_action[1])}}
   FORWARD();
   goto code_start;
 
   fell_through:
   CHECK(false);
 }
 } }