Experimental parser: user defined key classes

tools/lexer_generator/transition_keys.py

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 73 matching lines @@
     return TransitionKey.__cached_any
 
   @staticmethod
   def single_char(char):
     char = ord(char)
     assert (TransitionKey.__lower_bound <= char and
             char <= TransitionKey.__latin_1_upper_bound)
     return TransitionKey.__create([(char, char)])
 
   @staticmethod
-  def __process_graph(graph, ranges):
+  def __process_graph(graph, ranges, key_map):
     key = graph[0]
     if key == 'RANGE':
       ranges.append((ord(graph[1]), ord(graph[2])))
     elif key == 'LITERAL':
       ranges.append((ord(graph[1]), ord(graph[1])))
+    elif key == 'CAT':
+      for x in [graph[1], graph[2]]:
+        TransitionKey.__process_graph(x, ranges, key_map)
     elif key == 'CHARACTER_CLASS':
-      if graph[1] == 'ws':
+      class_name = graph[1]
+      if class_name == 'ws':
         ranges.append(TransitionKey.__unicode_whitespace_bounds)
-      elif graph[1] == 'lit':
+      elif class_name == 'lit':
         ranges.append(TransitionKey.__unicode_literal_bounds)
+      elif class_name in key_map:
+        ranges += key_map[class_name].__ranges
       else:
         raise Exception("unknown character class [%s]" % graph[1])
-    elif key == 'CAT':
-      for x in [graph[1], graph[2]]:
-        TransitionKey.__process_graph(x, ranges)
     else:
       raise Exception("bad key [%s]" % key)
 
   @staticmethod
-  def character_class(invert, graph):
+  def character_class(graph, key_map):
     ranges = []
-    TransitionKey.__process_graph(graph, ranges)
+    assert graph[0] == 'CLASS' or graph[0] == 'NOT_CLASS'
+    invert = graph[0] == 'NOT_CLASS'
+    TransitionKey.__process_graph(graph[1], ranges, key_map)
     return TransitionKey.__key_from_ranges(invert, ranges)
 
   def matches_char(self, char):
     char = ord(char)
     # TODO class checks
     for r in self.__ranges:
       if r[0] <= char and char <= r[1]: return True
     return False
 
   def matches_key(self, key):
@@ skipping 136 matching lines @@
       elif last[1] + 1 < r[0]:
         inverted.append((last[1] + 1, r[0] - 1))
       last = r
     if last != None and last[1] < TransitionKey.__latin_1_upper_bound:
       inverted.append((last[1] + 1, TransitionKey.__latin_1_upper_bound))
     if not contains_whitespace:
       inverted.append(TransitionKey.__unicode_whitespace_bounds)
     if not contains_literal:
       inverted.append(TransitionKey.__unicode_literal_bounds)
     return inverted