Experimental parser: user defined key classes

tools/lexer_generator/transition_keys.py

Index: tools/lexer_generator/transition_keys.py
diff --git a/tools/lexer_generator/transition_keys.py b/tools/lexer_generator/transition_keys.py
index d759a4f5b14f01aa4a3f10a40eef84c0e400a7bd..1c0de4d2b15b667477f52a3c2121c76d9b683c3f 100644
--- a/tools/lexer_generator/transition_keys.py
+++ b/tools/lexer_generator/transition_keys.py
@@ -91,29 +91,34 @@ class TransitionKey:
     return TransitionKey.__create([(char, char)])
 
   @staticmethod
-  def __process_graph(graph, ranges):
+  def __process_graph(graph, ranges, key_map):
     key = graph[0]
     if key == 'RANGE':
       ranges.append((ord(graph[1]), ord(graph[2])))
     elif key == 'LITERAL':
       ranges.append((ord(graph[1]), ord(graph[1])))
+    elif key == 'CAT':
+      for x in [graph[1], graph[2]]:
+        TransitionKey.__process_graph(x, ranges, key_map)
     elif key == 'CHARACTER_CLASS':
-      if graph[1] == 'ws':
+      class_name = graph[1]
+      if class_name == 'ws':
         ranges.append(TransitionKey.__unicode_whitespace_bounds)
-      elif graph[1] == 'lit':
+      elif class_name == 'lit':
         ranges.append(TransitionKey.__unicode_literal_bounds)
+      elif class_name in key_map:
+        ranges += key_map[class_name].__ranges
       else:
         raise Exception("unknown character class [%s]" % graph[1])
-    elif key == 'CAT':
-      for x in [graph[1], graph[2]]:
-        TransitionKey.__process_graph(x, ranges)
     else:
       raise Exception("bad key [%s]" % key)
 
   @staticmethod
-  def character_class(invert, graph):
+  def character_class(graph, key_map):
     ranges = []
-    TransitionKey.__process_graph(graph, ranges)
+    assert graph[0] == 'CLASS' or graph[0] == 'NOT_CLASS'
+    invert = graph[0] == 'NOT_CLASS'
+    TransitionKey.__process_graph(graph[1], ranges, key_map)
     return TransitionKey.__key_from_ranges(invert, ranges)
 
   def matches_char(self, char):