Experimental lexer generator: Lex files with zeros properly.

src/lexer/lexer_py.re

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 22 matching lines @@
 digit = [0-9];
 hex_digit = [0-9a-fA-F];
 single_escape_char = ['"\\bfnrtv];
 maybe_exponent = /([eE][\-+]?[:digit:]+)?/;
 number =
   /0[xX][:hex_digit:]+/ | (
   /\.[:digit:]+/ maybe_exponent |
   /[:digit:]+(\.[:digit:]*)?/ maybe_exponent );
 # TODO this is incomplete/incorrect
 line_terminator_sequence = (/\n\r?/)|(/\r\n?/);
+eof = [:eof:];
 
 # grammar is
 #   regex <action_on_state_entry|action_on_match|transition>
 #
 # actions can be c code enclosed in {} or identifiers to be passed to codegen
 # transition must be in continue or the name of a subgraph
 
 <<default>>
 "|="          <|push_token(ASSIGN_BIT_OR)|>
 "^="          <|push_token(ASSIGN_BIT_XOR)|>
@@ skipping 176 matching lines @@
 eof <|skip_and_terminate|>
 catch_all <||continue>
 
 <<MultiLineComment>>
 /\*+\//             <|skip|>
 # TODO find a way to generate the below rule
 /\*+[^\/*]/       <||continue>
 line_terminator  <push_line_terminator||continue>
 eof <|{start_ = marker_; BACKWARD(); PUSH_TOKEN(Token::ILLEGAL);}|>
 catch_all        <||continue>