Experimental lexer generator: generate code for utf-16 character classes.

tools/lexer_generator/transition_keys.py

Index: tools/lexer_generator/transition_keys.py
diff --git a/tools/lexer_generator/transition_keys.py b/tools/lexer_generator/transition_keys.py
index f2035d4f4aecd476898b7879c165663ac412fbaf..b1ce6b8d7b7b0a1f6dd07067ae20367f44e2309d 100644
--- a/tools/lexer_generator/transition_keys.py
+++ b/tools/lexer_generator/transition_keys.py
@@ -41,9 +41,10 @@ class TransitionKey:
     # These are not real ranges; they just need to be separate from any real
     # ranges.
     'whitespace' : (256, 256),
-    'literal' : (257, 257),
-    'eos' : (258, 258),
-    'zero' : (259, 259),
+    'letter' : (257, 257),
+    'identifier_part_not_letter' : (258, 258),
+    'eos' : (259, 259),
+    'zero' : (260, 260),
   }
   __lower_bound = 1
   __upper_bound = max(__class_bounds.values(), key=lambda item: item[1])[1]
@@ -139,14 +140,9 @@ class TransitionKey:
         TransitionKey.__process_graph(x, ranges, key_map)
     elif key == 'CHARACTER_CLASS':
       class_name = graph[1]
-      if class_name == 'ws':
-        ranges.append(TransitionKey.__class_bounds['whitespace'])
-      elif class_name == 'lit':
-        ranges.append(TransitionKey.__class_bounds['literal'])
-      elif class_name == 'eos':
-        ranges.append(TransitionKey.__class_bounds['eos'])
-      elif class_name == 'zero':
-        ranges.append(TransitionKey.__class_bounds['zero'])
+      if class_name in ['whitespace', 'letter', 'identifier_part_not_letter',
+                        'eos', 'zero']:

[dcarney, 2013/11/20 16:05:18]

class_name in __class_bounds.keys()?

+        ranges.append(TransitionKey.__class_bounds[class_name])
       elif class_name in key_map:
         ranges += key_map[class_name].__ranges
       else: