Experimental parser: add utf8 encoding

tools/lexer_generator/dfa.py

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 29 matching lines @@
   def transitions_to_multiple_states(self):
     return False
 
   def name(self):
     return self.__name
 
   def action(self):
     return self.__action
 
   def add_transition(self, key, state):
+    assert key != None
     assert not key == TransitionKey.epsilon()
     assert not self.__transitions.has_key(key)
     self.__transitions[key] = state
 
   def transitions(self):
     return self.__transitions
 
   def epsilon_closure_iter(self):
     return iter([])
 
@@ skipping 324 matching lines @@
           elif len(intersection) <= len(difference):
             working_set.add(intersection)
           else:
             working_set.add(difference)
         if old_partitions:
           partitions -= old_partitions
         if new_partitions:
           partitions |= new_partitions
     (start_name, mapping) = self.__create_states_from_partitions(partitions)
     return Dfa(self.__dfa.encoding(), start_name, mapping)