Experimental lexer generator: generate code for utf-16 character classes.

tools/lexer_generator/generator.py

Index: tools/lexer_generator/generator.py
diff --git a/tools/lexer_generator/generator.py b/tools/lexer_generator/generator.py
index 4e228304e553c502f9ffbb7fa45acdfc9496494f..fa0e37bf16aa36a911a533bf18200793f58c108f 100644
--- a/tools/lexer_generator/generator.py
+++ b/tools/lexer_generator/generator.py
@@ -97,7 +97,7 @@ if __name__ == '__main__':
   parser.add_argument('--re', default='src/lexer/lexer_py.re')
   parser.add_argument('--input')
   parser.add_argument('--code')
-  parser.add_argument('--char-type')
+  parser.add_argument('--encoding')
   parser.add_argument('--no-minimize-default', action='store_true')
   parser.add_argument('--no-verify-default', action='store_true')
   parser.add_argument('--no-inline', action='store_true')
@@ -133,13 +133,13 @@ if __name__ == '__main__':
         print "wrote html to %s" % html_file
 
   code_file = args.code
-  char_type = args.char_type
-  if not char_type:
-    char_type = 'uint8_t'
+  encoding = args.encoding
+  if not encoding:
+    encoding = 'latin1'
 
   if code_file:
     code_generator = CodeGenerator(rule_processor,
-                                   char_type,
+                                   encoding,
                                    minimize_default = minimize_default,
                                    log = verbose,
                                    inline = not args.no_inline,