Experimental lexer generator: generate code for utf-16 character classes.

tools/lexer_generator/transition_keys.py

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 23 matching lines @@
   (e.g., "whitespace"), defining for which characters the transition
   happens. When we generate code based on the transition key, the character
   ranges generate simple checks and the class ranges generate more complicated
   conditions, e.g., function calls.'''
 
   __class_bounds = {
     'latin_1' : (1, 255),
     # These are not real ranges; they just need to be separate from any real
     # ranges.
     'whitespace' : (256, 256),
-    'literal' : (257, 257),
-    'eos' : (258, 258),
-    'zero' : (259, 259),
+    'letter' : (257, 257),
+    'identifier_part_not_letter' : (258, 258),
+    'eos' : (259, 259),
+    'zero' : (260, 260),
   }
   __lower_bound = 1
   __upper_bound = max(__class_bounds.values(), key=lambda item: item[1])[1]
 
   __cached_keys = {}
 
   __unique_key_counter = -1
 
   @staticmethod
   def __in_latin_1(char):
@@ skipping 75 matching lines @@
     key = graph[0]
     if key == 'RANGE':
       ranges.append((ord(graph[1]), ord(graph[2])))
     elif key == 'LITERAL':
       ranges.append((ord(graph[1]), ord(graph[1])))
     elif key == 'CAT':
       for x in [graph[1], graph[2]]:
         TransitionKey.__process_graph(x, ranges, key_map)
     elif key == 'CHARACTER_CLASS':
       class_name = graph[1]
-      if class_name == 'ws':
-        ranges.append(TransitionKey.__class_bounds['whitespace'])
-      elif class_name == 'lit':
-        ranges.append(TransitionKey.__class_bounds['literal'])
-      elif class_name == 'eos':
-        ranges.append(TransitionKey.__class_bounds['eos'])
-      elif class_name == 'zero':
-        ranges.append(TransitionKey.__class_bounds['zero'])
+      if class_name in ['whitespace', 'letter', 'identifier_part_not_letter',
+                        'eos', 'zero']:

[dcarney, 2013/11/20 16:05:18]

class_name in __class_bounds.keys()?

+        ranges.append(TransitionKey.__class_bounds[class_name])
       elif class_name in key_map:
         ranges += key_map[class_name].__ranges
       else:
         raise Exception('unknown character class [%s]' % graph[1])
     else:
       raise Exception('bad key [%s]' % key)
 
   @staticmethod
   def character_class(graph, key_map):
     '''Processes 'graph' (a representation of a character class in the rule
@@ skipping 234 matching lines @@
       elif last[1] + 1 < r[0]:
         inverted.append((last[1] + 1, r[0] - 1))
       last = r
     upper_bound = latin_1[1]
     if last == None:
       inverted.append(latin_1)
     elif last[1] < upper_bound:
       inverted.append((last[1] + 1, upper_bound))
     inverted += list(classes)
     return inverted