Experimental parser: add utf8 encoding

tools/lexer_generator/code_generator.py

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 229 matching lines @@
     assert(default_action and default_action.match_action())
     default_action = default_action.match_action()
 
     sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
     template_env = jinja2.Environment(
       loader = jinja2.PackageLoader('lexer_generator', '.'),
       undefined = jinja2.StrictUndefined)
     template = template_env.get_template('code_generator.jinja')
 
     encoding = self.__dfa.encoding().name()
-    if encoding == 'latin1':
-      char_type = 'uint8_t'
-    elif encoding == 'utf16':
-      char_type = 'uint16_t'
-    else:
-      raise Exception('Unsupported encoding %s' % encoding)
+    char_types = {'latin1': 'uint8_t', 'utf16': 'uint16_t', 'utf8': 'int8_t'}
+    char_type = char_types[encoding]
+
     return template.render(
       start_node_number = 0,
       debug_print = self.__debug_print,
       default_action = default_action,
       dfa_states = dfa_states,
       encoding = encoding,
       char_type = char_type)