Experimental lexer generator: generate code for utf-16 character classes.

tools/lexer_generator/code_generator.py

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 17 matching lines @@
 import os
 import sys
 import jinja2
 from dfa import Dfa
 from transition_keys import TransitionKey
 
 class CodeGenerator:
 
   def __init__(self,
                rule_processor,
-               char_type,
+               encoding = 'latin1',
                minimize_default = True,
                inline = True,
                switching = True,
                debug_print = False,
                log = False):
     if minimize_default:
       dfa = rule_processor.default_automata().minimal_dfa()
     else:
       dfa = rule_processor.default_automata().dfa()
     self.__dfa = dfa
     self.__default_action = rule_processor.default_action
     self.__debug_print = debug_print
     self.__start_node_number = self.__dfa.start_state().node_number()
     self.__log = log
     self.__inline = inline
     self.__switching = switching
-    self.__char_type = char_type
+    self.__encoding = encoding
 
   def __state_cmp(self, left, right):
     if left['original_node_number'] == self.__start_node_number:
       return -1
     if right['original_node_number'] == self.__start_node_number:
       return 1
     c = cmp(len(left['disjoint_keys']), len(right['disjoint_keys']))
     if c:
       return c
     c = cmp(left['disjoint_keys'], right['disjoint_keys'])
@@ skipping 169 matching lines @@
     default_action = self.__default_action
     assert(default_action and default_action.match_action())
     default_action = default_action.match_action()
 
     sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
     template_env = jinja2.Environment(
       loader = jinja2.PackageLoader('lexer_generator', '.'),
       undefined = jinja2.StrictUndefined)
     template = template_env.get_template('code_generator.jinja')
 
+    if self.__encoding == 'latin1':
+      char_type = 'uint8_t'
+    elif self.__encoding == 'utf16':
+      char_type = 'uint16_t'
+    else:
+      raise Exception('Unsupported encoding %s' % encoding)
     return template.render(
       start_node_number = 0,
       debug_print = self.__debug_print,
       default_action = default_action,
       dfa_states = dfa_states,
-      char_type = self.__char_type)
+      encoding = self.__encoding,
+      char_type = char_type)