Experimental parser: cleanup code requiring calls to the unicode cache

tools/lexer_generator/transition_keys.py

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 39 matching lines @@
     self.__lower_bound = primary_range[0]
     self.__upper_bound = primary_range[1] + len(class_names)
     f = lambda i : (i + primary_range[1] + 1, i + primary_range[1] + 1)
     self.__class_ranges = {name : f(i) for i, name in enumerate(class_names)}
     self.__predefined_ranges = {}
 
   def name(self):
     return self.__name
 
   def add_predefined_range(self, name, ranges):
-    # TODO verify disjointness
     self.__predefined_ranges[name] = ranges
 
   def lower_bound(self):
     return self.__lower_bound
 
   def upper_bound(self):
     return self.__upper_bound
 
   def primary_range(self):
     return self.__primary_range
 
   def class_range(self, name):
     ranges = self.__class_ranges
     return None if not name in ranges else ranges[name]
 
   def class_range_iter(self):
     return self.__class_ranges.iteritems()
 
+  def class_name_iter(self):
+    return self.__class_ranges.iterkeys()
+
   def class_value_iter(self):
     return self.__class_ranges.itervalues()
 
   def predefined_range_iter(self, name):
     ranges = self.__predefined_ranges
     return None if not name in ranges else iter(ranges[name])
 
   def is_primary_range(self, r):
     assert self.lower_bound() <= r[0] and r[1] <= self.upper_bound()
     primary_range = self.__primary_range
@@ skipping 367 matching lines @@
       inverted.append((last[1] + 1, upper_bound))
     inverted += list(classes)
     return inverted
 
 class Latin1Encoding(KeyEncoding):
 
   def __init__(self):
     super(Latin1Encoding, self).__init__(
       'latin1',
       (1, 255),
-      ['eos', 'zero', 'byte_order_mark'])
+      ['eos', 'zero'])
     self.add_predefined_range(
       'whitespace', [(9, 9), (11, 12), (32, 32), (133, 133), (160, 160)])
     self.add_predefined_range(
       'letter', [
         (65, 90), (97, 122), (170, 170), (181, 181),
         (186, 186), (192, 214), (216, 246), (248, 255)])
     self.add_predefined_range('line_terminator', [(10, 10), (13, 13)])
     self.add_predefined_range(
       'identifier_part_not_letter', [(48, 57), (95, 95)])
 
 class Utf16Encoding(KeyEncoding):
 
   def __init__(self):
     super(Utf16Encoding, self).__init__(
       'utf16',
       (1, 255),
       ['eos', 'zero', 'byte_order_mark',
-       'non_latin_1_whitespace',
-       'non_latin_1_letter',
-       'non_latin_1_identifier_part_not_letter',
-       'non_latin_1_line_terminator',
-       'non_latin_1_everything_else'])
+       'non_primary_whitespace',
+       'non_primary_letter',
+       'non_primary_identifier_part_not_letter',
+       'non_primary_line_terminator',
+       'non_primary_everything_else'])
     self.add_predefined_range(
       'whitespace',
       [(9, 9), (11, 12), (32, 32), (133, 133), (160, 160),
-       self.class_range('non_latin_1_whitespace')])
+       self.class_range('byte_order_mark'),
+       self.class_range('non_primary_whitespace')])
     self.add_predefined_range(
       'letter', [
         (65, 90), (97, 122), (170, 170), (181, 181),
         (186, 186), (192, 214), (216, 246), (248, 255),
-        self.class_range('non_latin_1_letter')])
+        self.class_range('non_primary_letter')])
     self.add_predefined_range(
       'line_terminator',
-      [(10, 10), (13, 13), self.class_range('non_latin_1_line_terminator')])
+      [(10, 10), (13, 13), self.class_range('non_primary_line_terminator')])
     self.add_predefined_range(
       'identifier_part_not_letter',
       [(48, 57), (95, 95),
-       self.class_range('non_latin_1_identifier_part_not_letter')])
+       self.class_range('non_primary_identifier_part_not_letter')])
 
 class Utf8Encoding(KeyEncoding):
 
   def __init__(self):
     super(Utf8Encoding, self).__init__(
       'utf8',
       (1, 127),
       ['eos', 'zero', 'byte_order_mark',
-       'non_ascii_whitespace',
-       'non_ascii_letter',
-       'non_ascii_identifier_part_not_letter',
-       'non_ascii_line_terminator',
-       'non_ascii_everything_else'])
+       'non_primary_whitespace',
+       'non_primary_letter',
+       'non_primary_identifier_part_not_letter',
+       'non_primary_line_terminator',
+       'non_primary_everything_else'])
     self.add_predefined_range(
       'whitespace',
-      [(9, 9), (11, 12), (32, 32), self.class_range('non_ascii_whitespace')])
+      [(9, 9), (11, 12), (32, 32),
+        self.class_range('byte_order_mark'),
+        self.class_range('non_primary_whitespace')])
     self.add_predefined_range(
-      'letter', [(65, 90), (97, 122), self.class_range('non_ascii_letter')])
+      'letter', [(65, 90), (97, 122), self.class_range('non_primary_letter')])
     self.add_predefined_range(
       'line_terminator',
-      [(10, 10), (13, 13), self.class_range('non_ascii_line_terminator')])
+      [(10, 10), (13, 13), self.class_range('non_primary_line_terminator')])
     self.add_predefined_range(
       'identifier_part_not_letter',
       [(48, 57), (95, 95),
-       self.class_range('non_ascii_identifier_part_not_letter')])
+       self.class_range('non_primary_identifier_part_not_letter')])