Experimental lexer generator: Lex files with zeros properly.

tools/lexer_generator/transition_keys.py

Index: tools/lexer_generator/transition_keys.py
diff --git a/tools/lexer_generator/transition_keys.py b/tools/lexer_generator/transition_keys.py
index f639d43843620dd4417c6639edbde68380eaf0a2..4e650969f1ae8d866c6b99c32d433d46b6d14b92 100644
--- a/tools/lexer_generator/transition_keys.py
+++ b/tools/lexer_generator/transition_keys.py
@@ -31,8 +31,10 @@ class TransitionKey:
 
   __class_bounds = {
     "latin_1" : (0, 255),
+    # These are not "real" ranges; they just need to be separate.
     "whitespace" : (256, 256),
     "literal" : (257, 257),
+    "eof" : (258, 258),
   }
   __lower_bound = 0
   __upper_bound = reduce(lambda acc, (k, v): max(acc, v[1]), __class_bounds.items(), 0)
@@ -136,6 +138,8 @@ class TransitionKey:
         ranges.append(TransitionKey.__class_bounds["whitespace"])
       elif class_name == 'lit':
         ranges.append(TransitionKey.__class_bounds["literal"])
+      elif class_name == 'eof':
+        ranges.append(TransitionKey.__class_bounds["eof"])
       elif class_name in key_map:
         ranges += key_map[class_name].__ranges
       else: