Experimental parser: split classes into latin1 and non latin1

tools/lexer_generator/transition_keys.py

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 22 matching lines @@
   A transition key has a list of character ranges and a list of class ranges
   (e.g., "whitespace"), defining for which characters the transition
   happens. When we generate code based on the transition key, the character
   ranges generate simple checks and the class ranges generate more complicated
   conditions, e.g., function calls.'''
 
   __class_bounds = {
     'latin_1' : (1, 255),
     # These are not real ranges; they just need to be separate from any real
     # ranges.
-    'whitespace' : (256, 256),
-    'letter' : (257, 257),
-    'identifier_part_not_letter' : (258, 258),
-    'eos' : (259, 259),
-    'zero' : (260, 260),
+    'non_latin_1_whitespace' : (256, 256),
+    'non_latin_1_letter' : (257, 257),
+    'non_latin1_identifier_part_not_letter' : (258, 258),
+    'non_latin1_line_terminator' : (259, 259),
+    'eos' : (260, 260),
+    'zero' : (261, 261),
   }
-  __lower_bound = 1
+  __lower_bound = min(__class_bounds.values(), key=lambda item: item[0])[0]
   __upper_bound = max(__class_bounds.values(), key=lambda item: item[1])[1]
 
   __cached_keys = {}
 
   __unique_key_counter = -1
 
+  __predefined_ranges = {
+    'whitespace' : [
+        (9, 9), (11, 12), (32, 32), (133, 133), (160, 160),
+        __class_bounds['non_latin_1_whitespace']],
+    'letter' : [
+        (65, 90), (97, 122), (170, 170), (181, 181),
+        (186, 186), (192, 214), (216, 246), (248, 255),
+        __class_bounds['non_latin_1_letter']],
+    'line_terminator' : [
+        (10, 10), (13, 13),
+        __class_bounds['non_latin1_line_terminator']],
+    'identifier_part_not_letter' : [
+        (48, 57), (95, 95),
+        __class_bounds['non_latin1_identifier_part_not_letter']],
+  }
+
   @staticmethod
   def __in_latin_1(char):
     bound = TransitionKey.__class_bounds['latin_1']
     return (bound[0] <= char and char <= bound[1])
 
   @staticmethod
   def __is_class_range(r):
     return r[0] == r[1] and not TransitionKey.__in_latin_1(r[0])
 
   @staticmethod
@@ skipping 67 matching lines @@
     key = graph[0]
     if key == 'RANGE':
       ranges.append((ord(graph[1]), ord(graph[2])))
     elif key == 'LITERAL':
       ranges.append((ord(graph[1]), ord(graph[1])))
     elif key == 'CAT':
       for x in [graph[1], graph[2]]:
         TransitionKey.__process_graph(x, ranges, key_map)
     elif key == 'CHARACTER_CLASS':
       class_name = graph[1]
-      if class_name in TransitionKey.__class_bounds.keys():
+      if class_name in TransitionKey.__class_bounds:
+        if class_name in key_map:
+          assert (key_map[class_name] ==
+              TransitionKey([TransitionKey.__class_bounds[class_name]]))
         ranges.append(TransitionKey.__class_bounds[class_name])
+      elif class_name in TransitionKey.__predefined_ranges:
+        if class_name in key_map:
+          assert (key_map[class_name] ==
+              TransitionKey(TransitionKey.__predefined_ranges[class_name]))
+        ranges += TransitionKey.__predefined_ranges[class_name]
       elif class_name in key_map:
         ranges += key_map[class_name].__ranges
       else:
         raise Exception('unknown character class [%s]' % graph[1])
     else:
       raise Exception('bad key [%s]' % key)
 
   @staticmethod
   def character_class(graph, key_map):
     '''Processes 'graph' (a representation of a character class in the rule
@@ skipping 234 matching lines @@
       elif last[1] + 1 < r[0]:
         inverted.append((last[1] + 1, r[0] - 1))
       last = r
     upper_bound = latin_1[1]
     if last == None:
       inverted.append(latin_1)
     elif last[1] < upper_bound:
       inverted.append((last[1] + 1, upper_bound))
     inverted += list(classes)
     return inverted