Experimental parser: add utf8 encoding

tools/lexer_generator/transition_keys.py

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 19 matching lines @@
 
 class KeyEncoding(object):
 
   __encodings = {}
 
   @staticmethod
   def get(name):
     if not KeyEncoding.__encodings:
       Latin1Encoding()
       Utf16Encoding()
+      Utf8Encoding()
     return KeyEncoding.__encodings[name]
 
   def __init__(self, name, primary_range, class_names):
     assert not name in KeyEncoding.__encodings
     KeyEncoding.__encodings[name] = self
     assert primary_range[0] <= primary_range[1]
     assert primary_range[0] >= 0
     self.__name = name
     self.__primary_range = primary_range
     self.__lower_bound = primary_range[0]
@@ skipping 444 matching lines @@
         (65, 90), (97, 122), (170, 170), (181, 181),
         (186, 186), (192, 214), (216, 246), (248, 255),
         self.class_range('non_latin_1_letter')])
     self.add_predefined_range(
       'line_terminator',
       [(10, 10), (13, 13), self.class_range('non_latin_1_line_terminator')])
     self.add_predefined_range(
       'identifier_part_not_letter',
       [(48, 57), (95, 95),
        self.class_range('non_latin_1_identifier_part_not_letter')])
+
+class Utf8Encoding(KeyEncoding):
+
+  def __init__(self):
+    super(Utf8Encoding, self).__init__(
+      'utf8',
+      (1, 127),
+      ['eos', 'zero', 'byte_order_mark',
+       'non_ascii_whitespace',
+       'non_ascii_letter',
+       'non_ascii_identifier_part_not_letter',
+       'non_ascii_line_terminator',
+       'non_ascii_everything_else'])
+    self.add_predefined_range(
+      'whitespace',
+      [(9, 9), (11, 12), (32, 32), self.class_range('non_ascii_whitespace')])
+    self.add_predefined_range(
+      'letter', [(65, 90), (97, 122), self.class_range('non_ascii_letter')])
+    self.add_predefined_range(
+      'line_terminator',
+      [(10, 10), (13, 13), self.class_range('non_ascii_line_terminator')])
+    self.add_predefined_range(
+      'identifier_part_not_letter',
+      [(48, 57), (95, 95),
+       self.class_range('non_ascii_identifier_part_not_letter')])