Experimental parser: add utf8 encoding

tools/lexer_generator/nfa.py

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 44 matching lines @@
   def set_action(self, action):
     assert not self.is_closed()
     assert not self.__action
     self.__action = action
 
   def transitions(self):
     assert self.is_closed()
     return self.__transitions
 
   def __add_transition(self, key, next_state):
+    assert key != None
     if next_state == None:
       assert not self.is_closed(), "already closed"
       self.__unclosed.add(key)
       return
     if not key in self.__transitions:
       self.__transitions[key] = set()
     self.__transitions[key].add(next_state)
 
   def add_unclosed_transition(self, key):
     assert key != TransitionKey.epsilon()
@@ skipping 71 matching lines @@
       f = lambda state: state.transition_state_iter_for_key(key)
       match_states |= set(chain(*map(f, nfa_state_set)))
       transition_state = self.__to_dfa(match_states, dfa_nodes, end_node)
       dfa_nodes[name]['transitions'][key] = transition_state
     return name
 
   def compute_dfa(self):
     dfa_nodes = {}
     start_name = self.__to_dfa(set([self.__start]), dfa_nodes, self.__end)
     return (start_name, dfa_nodes)