Experimental lexer generator: Fix byte order mark detection.

tools/lexer_generator/transition_keys.py

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 28 matching lines @@
   __class_bounds = {
     'latin_1' : (1, 255),
     # These are not real ranges; they just need to be separate from any real
     # ranges.
     'non_latin_1_whitespace' : (256, 256),
     'non_latin_1_letter' : (257, 257),
     'non_latin1_identifier_part_not_letter' : (258, 258),
     'non_latin1_line_terminator' : (259, 259),
     'eos' : (260, 260),
     'zero' : (261, 261),
+    'byte_order_mark' : (262, 262),
   }
   __lower_bound = min(__class_bounds.values(), key=lambda item: item[0])[0]
   __upper_bound = max(__class_bounds.values(), key=lambda item: item[1])[1]
 
   __cached_keys = {}
 
   __unique_key_counter = -1
 
   __predefined_ranges = {
     'whitespace' : [
@@ skipping 355 matching lines @@
       elif last[1] + 1 < r[0]:
         inverted.append((last[1] + 1, r[0] - 1))
       last = r
     upper_bound = latin_1[1]
     if last == None:
       inverted.append(latin_1)
     elif last[1] < upper_bound:
       inverted.append((last[1] + 1, upper_bound))
     inverted += list(classes)
     return inverted