Experimental lexer generator: Lex files with zeros properly.

tools/lexer_generator/transition_keys.py

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 13 matching lines @@
 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 from string import printable
 
 class TransitionKey:
 
   __class_bounds = {
     "latin_1" : (0, 255),
+    # These are not "real" ranges; they just need to be separate.
     "whitespace" : (256, 256),
     "literal" : (257, 257),
+    "eof" : (258, 258),
   }
   __lower_bound = 0
   __upper_bound = reduce(lambda acc, (k, v): max(acc, v[1]), __class_bounds.items(), 0)
 
   __cached_keys = {}
 
   __unique_key_counter = -1
 
   @staticmethod
   def __in_latin_1(char):
@@ skipping 83 matching lines @@
       ranges.append((ord(graph[1]), ord(graph[1])))
     elif key == 'CAT':
       for x in [graph[1], graph[2]]:
         TransitionKey.__process_graph(x, ranges, key_map)
     elif key == 'CHARACTER_CLASS':
       class_name = graph[1]
       if class_name == 'ws':
         ranges.append(TransitionKey.__class_bounds["whitespace"])
       elif class_name == 'lit':
         ranges.append(TransitionKey.__class_bounds["literal"])
+      elif class_name == 'eof':
+        ranges.append(TransitionKey.__class_bounds["eof"])
       elif class_name in key_map:
         ranges += key_map[class_name].__ranges
       else:
         raise Exception("unknown character class [%s]" % graph[1])
     else:
       raise Exception("bad key [%s]" % key)
 
   @staticmethod
   def character_class(graph, key_map):
     ranges = []
@@ skipping 197 matching lines @@
       elif last[1] + 1 < r[0]:
         inverted.append((last[1] + 1, r[0] - 1))
       last = r
     upper_bound = latin_1[1]
     if last == None:
       inverted.append(latin_1)
     elif last[1] < upper_bound:
       inverted.append((last[1] + 1, upper_bound))
     inverted += list(classes)
     return inverted