Experimental lexer generator: generate code for utf-16 character classes.

tools/lexer_generator/generator.py

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 79 matching lines @@
   for t in rule_processor.default_automata().dfa().lex(string + '\0'):
     print t
 
 if __name__ == '__main__':
 
   parser = argparse.ArgumentParser()
   parser.add_argument('--html')
   parser.add_argument('--re', default='src/lexer/lexer_py.re')
   parser.add_argument('--input')
   parser.add_argument('--code')
-  parser.add_argument('--char-type')
+  parser.add_argument('--encoding')
   parser.add_argument('--no-minimize-default', action='store_true')
   parser.add_argument('--no-verify-default', action='store_true')
   parser.add_argument('--no-inline', action='store_true')
   parser.add_argument('--verbose', action='store_true')
   parser.add_argument('--debug-code', action='store_true')
   args = parser.parse_args()
 
   minimize_default = not args.no_minimize_default
   verbose = args.verbose
 
@@ skipping 15 matching lines @@
 
   html_file = args.html
   if html_file:
     html = generate_html(rule_processor, minimize_default)
     with open(args.html, 'w') as f:
       f.write(html)
       if verbose:
         print "wrote html to %s" % html_file
 
   code_file = args.code
-  char_type = args.char_type
-  if not char_type:
-    char_type = 'uint8_t'
+  encoding = args.encoding
+  if not encoding:
+    encoding = 'latin1'
 
   if code_file:
     code_generator = CodeGenerator(rule_processor,
-                                   char_type,
+                                   encoding,
                                    minimize_default = minimize_default,
                                    log = verbose,
                                    inline = not args.no_inline,
                                    debug_print = args.debug_code)
     code = code_generator.process()
     with open(code_file, 'w') as f:
       f.write(code)
       if verbose:
         print "wrote code to %s" % code_file
 
   input_file = args.input
   if input_file:
     with open(input_file, 'r') as f:
       lex(rule_processor, f.read())